fm_size=(32, 32),
batch_size=batch_size,
border_mode="same")
model = ConvAutoEncoder(
layers=[layer_0, layer_1, layer_2,
MaxPoolingSameSize((32, 32)), layer_3])
#model=ConvAutoEncoder(layers=[layer_0, layer_3]);
out = model.fprop(images, corruption_level=None)
cost = mean_square_cost(out[-1], images)
#+L2_regularization(model.params, 0.005);
updates = gd_updates(cost=cost,
params=model.params,
method="sgd",
learning_rate=0.001,
momentum=0.9)
train = theano.function(
inputs=[idx],
outputs=[cost],
updates=updates,
givens={X: train_set_x[idx * batch_size:(idx + 1) * batch_size]})
print "[MESSAGE] The model is built"
epoch = 0
while (epoch < n_epochs):
epoch = epoch + 1
c = []
X=T.matrix("data");
y=T.ivector("label");
idx=T.lscalar();
layers=[ReLULayer(in_dim=784, out_dim=50)];
for i in xrange(20):
layers.append(HighwayReLULayer(in_dim=50));
layers.append(SoftmaxLayer(in_dim=50, out_dim=10));
model=FeedForward(layers=layers);
out=model.fprop(X);
cost=categorical_cross_entropy_cost(out[-1], y);
updates=gd_updates(cost=cost, params=model.params, method="sgd", learning_rate=0.01, momentum=0.9);
train=theano.function(inputs=[idx],
outputs=cost,
updates=updates,
givens={X: train_set_x[idx * batch_size: (idx + 1) * batch_size],
y: train_set_y[idx * batch_size: (idx + 1) * batch_size]});
test=theano.function(inputs=[idx],
outputs=model.layers[-1].error(out[-1], y),
givens={X: test_set_x[idx * batch_size: (idx + 1) * batch_size],
y: test_set_y[idx * batch_size: (idx + 1) * batch_size]});
print "[MESSAGE] The model is built"
def dconvae_experiment(nkerns,
n_epochs,
batch_size,
start_noise,
noise_step):
""" Setup dConvAE experiment
This experiment is setting up for testing
multi level noises on dConvAE using CIFAR-10
Parameters
----------
nkerns : int
number of filters
n_epochs : int
number of training epochs
batch_size : int
size of each training batch
start_noise : float
start level of noise
noise_step : float
incremental steps of noises
"""
# Xtr, Ytr, _, _=ds.load_CIFAR10("../data/CIFAR10");
#
# Xtr=np.mean(Xtr, 3);
# # Xte=np.mean(Xte, 3);
# Xtrain=Xtr.reshape(Xtr.shape[0], Xtr.shape[1]*Xtr.shape[2])/255.0;
# # Xtest=Xte.reshape(Xte.shape[0], Xte.shape[1]*Xte.shape[2])/255.0;
#
# train_set_x, _=ds.shared_dataset((Xtrain, Ytr));
# # test_set_x, test_set_y=ds.shared_dataset((Xtest, Yte));
#
# n_train_batches=train_set_x.get_value(borrow=True).shape[0]/batch_size;
# # n_test_batches=test_set_x.get_value(borrow=True).shape[0]/batch_size;
datasets=ds.load_mnist("../data/mnist.pkl.gz");
train_set_x, train_set_y = datasets[0];
valid_set_x, valid_set_y = datasets[1];
test_set_x, test_set_y = datasets[2];
n_train_batches=train_set_x.get_value(borrow=True).shape[0]/batch_size;
n_valid_batches=valid_set_x.get_value(borrow=True).shape[0]/batch_size;
n_test_batches=test_set_x.get_value(borrow=True).shape[0]/batch_size;
print "[MESSAGE] The data is loaded"
X=T.matrix("data");
# y=T.ivector("label");
idx=T.lscalar();
corruption_level=T.fscalar();
images=X.reshape((batch_size, 1, 28, 28))
layer_0=ReLUConvLayer(filter_size=(5,5),
num_filters=nkerns,
num_channels=1,
fm_size=(28,28),
batch_size=batch_size,
border_mode="same");
layer_1=IdentityConvLayer(filter_size=(11,11),
num_filters=1,
num_channels=nkerns,
fm_size=(28,28),
batch_size=batch_size,
border_mode="same");
model=ConvAutoEncoder(layers=[layer_0, MaxPoolingSameSize((28,28)), layer_1]);
out=model.fprop(images, corruption_level=corruption_level);
cost=mean_square_cost(out[-1], images)#+L2_regularization(model.params, 0.005);
updates=gd_updates(cost=cost, params=model.params, method="sgd", learning_rate=0.001, momentum=0.975);
train=theano.function(inputs=[idx, corruption_level],
outputs=[cost],
updates=updates,
givens={X: train_set_x[idx * batch_size: (idx + 1) * batch_size]});
print "[MESSAGE] The model is built"
epoch = 0;
min_cost=None;
corr=np.random.uniform(low=start_noise, high=start_noise+0.1, size=1).astype("float32");
corr_best=corr[0];
max_iter=0;
while (epoch < n_epochs):
epoch = epoch + 1;
c = []
for batch_index in xrange(n_train_batches):
train_cost=train(batch_index, corr_best)
c.append(train_cost);
if min_cost==None:
min_cost=np.mean(c);
else:
if (np.mean(c)<min_cost*0.5) or (max_iter>=30):
min_cost=np.mean(c);
corr=np.random.uniform(low=corr_best, high=corr_best+noise_step, size=1).astype("float32");
corr_best=corr[0]
max_iter=0;
else:
max_iter+=1;
print 'Training epoch %d, cost %f, min cost %f, curr best %f, curr iter %d' % (epoch, np.mean(c), min_cost, corr_best, max_iter);
filters=model.layers[-1].filters.get_value(borrow=True);
for i in xrange(nkerns):
image_adr="../data/dConvAE_multi_level/dConvAE_multi_level_%d.eps" % (i);
plt.imshow(filters[0, i, :, :], cmap = plt.get_cmap('gray'), interpolation='nearest');
plt.axis('off');
plt.savefig(image_adr , bbox_inches='tight', pad_inches=0);
layer_3 = SoftmaxLayer(in_dim=200, out_dim=10)
#dropout=multi_dropout([(batch_size, 1, 28, 28), None, (batch_size, 50, 11, 11), None, None, None, None], prob=0.5);
dropout = multi_dropout([(batch_size, 1, 28, 28), None,
(batch_size, 50, 11, 11), None, None, None, None],
prob=0.5)
model = FeedForward(
layers=[layer_0, pool_0, layer_1, pool_1, flattener, layer_2, layer_3],
dropout=dropout)
out = model.fprop(images)
cost = categorical_cross_entropy_cost(out[-1], y) + L2_regularization(
model.params, 0.01)
updates = gd_updates(cost=cost,
params=model.params,
method="sgd",
learning_rate=0.1)
train = theano.function(
inputs=[idx],
outputs=cost,
updates=updates,
givens={
X: train_set_x[idx * batch_size:(idx + 1) * batch_size],
y: train_set_y[idx * batch_size:(idx + 1) * batch_size]
})
test = theano.function(inputs=[idx],
outputs=model.layers[-1].error(out[-1], y),
givens={
X:
layer_0_de = IdentityConvLayer(filter_size=(7, 7),
num_filters=1,
num_channels=64,
fm_size=(32, 32),
batch_size=batch_size,
border_mode="same")
model = ConvAutoEncoder(layers=[layer_0_en, layer_0_de])
out = model.fprop(images, corruption_level=corruption_level)
cost = mean_square_cost(out[-1],
images) #+L2_regularization(model.params, 0.005);
updates = gd_updates(cost=cost,
params=model.params,
method="sgd",
learning_rate=0.01,
momentum=0.9)
train = theano.function(
inputs=[idx, corruption_level],
outputs=[cost],
updates=updates,
givens={X: train_set_x[idx * batch_size:(idx + 1) * batch_size]})
print "[MESSAGE] The Layer 0 model is built"
epoch = 0
corr = np.random.uniform(low=0.1, high=0.2, size=1).astype("float32")
min_cost = None
corr_best = corr[0]
print "[MESSAGE] The data is loaded"
X = T.matrix("data")
idx = T.lscalar()
noise = theano.shared(np.asarray(np.random.normal(scale=0.1,
size=(batch_size, 784)),
dtype="float32"),
borrow=True)
#corrupted=corrupt_input(X, corruption_level=noise, noise_type="gaussian");
corrupted = X + noise
cost = binary_cross_entropy_cost(corrupted, X)
updates = gd_updates(cost, [noise], method="sgd", learning_rate=0.001)
train = theano.function(
inputs=[idx],
outputs=[cost],
updates=updates,
givens={X: train_set_x[idx * batch_size:(idx + 1) * batch_size]})
epoch = 0
while (epoch < n_epochs):
epoch = epoch + 1
c = []
for batch_index in xrange(n_train_batches):
train_cost = train(batch_index)
c.append(train_cost)
ep_idx=T.lscalar();
corruption_level=T.fscalar();
encode_layer=SigmoidLayer(in_dim=784,
out_dim=500);
decode_layer=SigmoidLayer(in_dim=500,
out_dim=784);
model=AutoEncoder(layers=[encode_layer, decode_layer]);
#out=model.fprop(X, corruption_level=corruption_level, noise_type="gaussian");
out=model.fprop(X, corruption_level=corruption_level);
cost=binary_cross_entropy_cost(out[-1], X);
updates=gd_updates(cost=cost, params=model.params, method="sgd", learning_rate=0.1);
train=theano.function(inputs=[idx, corruption_level],
outputs=[cost],
updates=updates,
givens={X: train_set_x[idx * batch_size: (idx + 1) * batch_size]});
print "[MESSAGE] The model is built"
epoch = 0;
min_cost=None;
corr=np.random.uniform(low=0.2, high=0.3, size=1).astype("float32");
corr_best=corr[0]
while (epoch < n_epochs):
epoch = epoch + 1;
c = []
n_test_batches=test_set_x.get_value(borrow=True).shape[0]/batch_size;
print "[MESSAGE] The data is loaded"
X=T.matrix("data");
idx=T.lscalar();
noise=theano.shared(np.asarray(np.random.normal(scale=0.1, size=(batch_size, 784)),
dtype="float32"),
borrow=True);
#corrupted=corrupt_input(X, corruption_level=noise, noise_type="gaussian");
corrupted=X+noise;
cost=binary_cross_entropy_cost(corrupted, X);
updates=gd_updates(cost, [noise], method="sgd", learning_rate=0.001);
train=theano.function(inputs=[idx],
outputs=[cost],
updates=updates,
givens={X: train_set_x[idx * batch_size: (idx + 1) * batch_size]});
epoch = 0;
while (epoch < n_epochs):
epoch = epoch + 1;
c = [];
for batch_index in xrange(n_train_batches):
train_cost=train(batch_index);
c.append(train_cost);
#co.append(curr_corr);
idx=T.lscalar();
layers=[ReLULayer(in_dim=6, out_dim=30)];
for i in xrange(20):
layers.append(HighwayReLULayer(in_dim=30));
layers.append(ReLULayer(in_dim=30, out_dim=1));
model=FeedForward(layers=layers);
out=model.fprop(X);
cost=mean_squared_cost(out[-1], y)+L2_regularization(model.params, 0.01);
#cost=binary_cross_entropy_cost(out[-1], y)+L2_regularization(model.params, 0.01);
#updates=gd_updates(cost=cost, params=model.params, method="adagrad", learning_rate=0.01);
updates=gd_updates(cost=cost, params=model.params, method="rmsprop", learning_rate=0.001);
train=theano.function(inputs=[idx],
outputs=cost,
updates=updates,
givens={X: X_train[idx * batch_size: (idx + 1) * batch_size],
y: y_train[idx * batch_size: (idx + 1) * batch_size]});
test=theano.function(inputs=[idx],
outputs=cost,
givens={X: X_test[idx * batch_size: (idx + 1) * batch_size],
y: y_test[idx * batch_size: (idx + 1) * batch_size]});
print "[MESSAGE] The model is built"
test_record=np.zeros((n_epochs, 1));
