fm_size=(32, 32), batch_size=batch_size, border_mode="same") model = ConvAutoEncoder( layers=[layer_0, layer_1, layer_2, MaxPoolingSameSize((32, 32)), layer_3]) #model=ConvAutoEncoder(layers=[layer_0, layer_3]); out = model.fprop(images, corruption_level=None) cost = mean_square_cost(out[-1], images) #+L2_regularization(model.params, 0.005); updates = gd_updates(cost=cost, params=model.params, method="sgd", learning_rate=0.001, momentum=0.9) train = theano.function( inputs=[idx], outputs=[cost], updates=updates, givens={X: train_set_x[idx * batch_size:(idx + 1) * batch_size]}) print "[MESSAGE] The model is built" epoch = 0 while (epoch < n_epochs): epoch = epoch + 1 c = []
X=T.matrix("data"); y=T.ivector("label"); idx=T.lscalar(); layers=[ReLULayer(in_dim=784, out_dim=50)]; for i in xrange(20): layers.append(HighwayReLULayer(in_dim=50)); layers.append(SoftmaxLayer(in_dim=50, out_dim=10)); model=FeedForward(layers=layers); out=model.fprop(X); cost=categorical_cross_entropy_cost(out[-1], y); updates=gd_updates(cost=cost, params=model.params, method="sgd", learning_rate=0.01, momentum=0.9); train=theano.function(inputs=[idx], outputs=cost, updates=updates, givens={X: train_set_x[idx * batch_size: (idx + 1) * batch_size], y: train_set_y[idx * batch_size: (idx + 1) * batch_size]}); test=theano.function(inputs=[idx], outputs=model.layers[-1].error(out[-1], y), givens={X: test_set_x[idx * batch_size: (idx + 1) * batch_size], y: test_set_y[idx * batch_size: (idx + 1) * batch_size]}); print "[MESSAGE] The model is built"
def dconvae_experiment(nkerns, n_epochs, batch_size, start_noise, noise_step): """ Setup dConvAE experiment This experiment is setting up for testing multi level noises on dConvAE using CIFAR-10 Parameters ---------- nkerns : int number of filters n_epochs : int number of training epochs batch_size : int size of each training batch start_noise : float start level of noise noise_step : float incremental steps of noises """ # Xtr, Ytr, _, _=ds.load_CIFAR10("../data/CIFAR10"); # # Xtr=np.mean(Xtr, 3); # # Xte=np.mean(Xte, 3); # Xtrain=Xtr.reshape(Xtr.shape[0], Xtr.shape[1]*Xtr.shape[2])/255.0; # # Xtest=Xte.reshape(Xte.shape[0], Xte.shape[1]*Xte.shape[2])/255.0; # # train_set_x, _=ds.shared_dataset((Xtrain, Ytr)); # # test_set_x, test_set_y=ds.shared_dataset((Xtest, Yte)); # # n_train_batches=train_set_x.get_value(borrow=True).shape[0]/batch_size; # # n_test_batches=test_set_x.get_value(borrow=True).shape[0]/batch_size; datasets=ds.load_mnist("../data/mnist.pkl.gz"); train_set_x, train_set_y = datasets[0]; valid_set_x, valid_set_y = datasets[1]; test_set_x, test_set_y = datasets[2]; n_train_batches=train_set_x.get_value(borrow=True).shape[0]/batch_size; n_valid_batches=valid_set_x.get_value(borrow=True).shape[0]/batch_size; n_test_batches=test_set_x.get_value(borrow=True).shape[0]/batch_size; print "[MESSAGE] The data is loaded" X=T.matrix("data"); # y=T.ivector("label"); idx=T.lscalar(); corruption_level=T.fscalar(); images=X.reshape((batch_size, 1, 28, 28)) layer_0=ReLUConvLayer(filter_size=(5,5), num_filters=nkerns, num_channels=1, fm_size=(28,28), batch_size=batch_size, border_mode="same"); layer_1=IdentityConvLayer(filter_size=(11,11), num_filters=1, num_channels=nkerns, fm_size=(28,28), batch_size=batch_size, border_mode="same"); model=ConvAutoEncoder(layers=[layer_0, MaxPoolingSameSize((28,28)), layer_1]); out=model.fprop(images, corruption_level=corruption_level); cost=mean_square_cost(out[-1], images)#+L2_regularization(model.params, 0.005); updates=gd_updates(cost=cost, params=model.params, method="sgd", learning_rate=0.001, momentum=0.975); train=theano.function(inputs=[idx, corruption_level], outputs=[cost], updates=updates, givens={X: train_set_x[idx * batch_size: (idx + 1) * batch_size]}); print "[MESSAGE] The model is built" epoch = 0; min_cost=None; corr=np.random.uniform(low=start_noise, high=start_noise+0.1, size=1).astype("float32"); corr_best=corr[0]; max_iter=0; while (epoch < n_epochs): epoch = epoch + 1; c = [] for batch_index in xrange(n_train_batches): train_cost=train(batch_index, corr_best) c.append(train_cost); if min_cost==None: min_cost=np.mean(c); else: if (np.mean(c)<min_cost*0.5) or (max_iter>=30): min_cost=np.mean(c); corr=np.random.uniform(low=corr_best, high=corr_best+noise_step, size=1).astype("float32"); corr_best=corr[0] max_iter=0; else: max_iter+=1; print 'Training epoch %d, cost %f, min cost %f, curr best %f, curr iter %d' % (epoch, np.mean(c), min_cost, corr_best, max_iter); filters=model.layers[-1].filters.get_value(borrow=True); for i in xrange(nkerns): image_adr="../data/dConvAE_multi_level/dConvAE_multi_level_%d.eps" % (i); plt.imshow(filters[0, i, :, :], cmap = plt.get_cmap('gray'), interpolation='nearest'); plt.axis('off'); plt.savefig(image_adr , bbox_inches='tight', pad_inches=0);
layer_3 = SoftmaxLayer(in_dim=200, out_dim=10) #dropout=multi_dropout([(batch_size, 1, 28, 28), None, (batch_size, 50, 11, 11), None, None, None, None], prob=0.5); dropout = multi_dropout([(batch_size, 1, 28, 28), None, (batch_size, 50, 11, 11), None, None, None, None], prob=0.5) model = FeedForward( layers=[layer_0, pool_0, layer_1, pool_1, flattener, layer_2, layer_3], dropout=dropout) out = model.fprop(images) cost = categorical_cross_entropy_cost(out[-1], y) + L2_regularization( model.params, 0.01) updates = gd_updates(cost=cost, params=model.params, method="sgd", learning_rate=0.1) train = theano.function( inputs=[idx], outputs=cost, updates=updates, givens={ X: train_set_x[idx * batch_size:(idx + 1) * batch_size], y: train_set_y[idx * batch_size:(idx + 1) * batch_size] }) test = theano.function(inputs=[idx], outputs=model.layers[-1].error(out[-1], y), givens={ X:
layer_0_de = IdentityConvLayer(filter_size=(7, 7), num_filters=1, num_channels=64, fm_size=(32, 32), batch_size=batch_size, border_mode="same") model = ConvAutoEncoder(layers=[layer_0_en, layer_0_de]) out = model.fprop(images, corruption_level=corruption_level) cost = mean_square_cost(out[-1], images) #+L2_regularization(model.params, 0.005); updates = gd_updates(cost=cost, params=model.params, method="sgd", learning_rate=0.01, momentum=0.9) train = theano.function( inputs=[idx, corruption_level], outputs=[cost], updates=updates, givens={X: train_set_x[idx * batch_size:(idx + 1) * batch_size]}) print "[MESSAGE] The Layer 0 model is built" epoch = 0 corr = np.random.uniform(low=0.1, high=0.2, size=1).astype("float32") min_cost = None corr_best = corr[0]
print "[MESSAGE] The data is loaded" X = T.matrix("data") idx = T.lscalar() noise = theano.shared(np.asarray(np.random.normal(scale=0.1, size=(batch_size, 784)), dtype="float32"), borrow=True) #corrupted=corrupt_input(X, corruption_level=noise, noise_type="gaussian"); corrupted = X + noise cost = binary_cross_entropy_cost(corrupted, X) updates = gd_updates(cost, [noise], method="sgd", learning_rate=0.001) train = theano.function( inputs=[idx], outputs=[cost], updates=updates, givens={X: train_set_x[idx * batch_size:(idx + 1) * batch_size]}) epoch = 0 while (epoch < n_epochs): epoch = epoch + 1 c = [] for batch_index in xrange(n_train_batches): train_cost = train(batch_index) c.append(train_cost)
ep_idx=T.lscalar(); corruption_level=T.fscalar(); encode_layer=SigmoidLayer(in_dim=784, out_dim=500); decode_layer=SigmoidLayer(in_dim=500, out_dim=784); model=AutoEncoder(layers=[encode_layer, decode_layer]); #out=model.fprop(X, corruption_level=corruption_level, noise_type="gaussian"); out=model.fprop(X, corruption_level=corruption_level); cost=binary_cross_entropy_cost(out[-1], X); updates=gd_updates(cost=cost, params=model.params, method="sgd", learning_rate=0.1); train=theano.function(inputs=[idx, corruption_level], outputs=[cost], updates=updates, givens={X: train_set_x[idx * batch_size: (idx + 1) * batch_size]}); print "[MESSAGE] The model is built" epoch = 0; min_cost=None; corr=np.random.uniform(low=0.2, high=0.3, size=1).astype("float32"); corr_best=corr[0] while (epoch < n_epochs): epoch = epoch + 1; c = []
n_test_batches=test_set_x.get_value(borrow=True).shape[0]/batch_size; print "[MESSAGE] The data is loaded" X=T.matrix("data"); idx=T.lscalar(); noise=theano.shared(np.asarray(np.random.normal(scale=0.1, size=(batch_size, 784)), dtype="float32"), borrow=True); #corrupted=corrupt_input(X, corruption_level=noise, noise_type="gaussian"); corrupted=X+noise; cost=binary_cross_entropy_cost(corrupted, X); updates=gd_updates(cost, [noise], method="sgd", learning_rate=0.001); train=theano.function(inputs=[idx], outputs=[cost], updates=updates, givens={X: train_set_x[idx * batch_size: (idx + 1) * batch_size]}); epoch = 0; while (epoch < n_epochs): epoch = epoch + 1; c = []; for batch_index in xrange(n_train_batches): train_cost=train(batch_index); c.append(train_cost); #co.append(curr_corr);
idx=T.lscalar(); layers=[ReLULayer(in_dim=6, out_dim=30)]; for i in xrange(20): layers.append(HighwayReLULayer(in_dim=30)); layers.append(ReLULayer(in_dim=30, out_dim=1)); model=FeedForward(layers=layers); out=model.fprop(X); cost=mean_squared_cost(out[-1], y)+L2_regularization(model.params, 0.01); #cost=binary_cross_entropy_cost(out[-1], y)+L2_regularization(model.params, 0.01); #updates=gd_updates(cost=cost, params=model.params, method="adagrad", learning_rate=0.01); updates=gd_updates(cost=cost, params=model.params, method="rmsprop", learning_rate=0.001); train=theano.function(inputs=[idx], outputs=cost, updates=updates, givens={X: X_train[idx * batch_size: (idx + 1) * batch_size], y: y_train[idx * batch_size: (idx + 1) * batch_size]}); test=theano.function(inputs=[idx], outputs=cost, givens={X: X_test[idx * batch_size: (idx + 1) * batch_size], y: y_test[idx * batch_size: (idx + 1) * batch_size]}); print "[MESSAGE] The model is built" test_record=np.zeros((n_epochs, 1));