Esempio n. 1
0
                            fm_size=(32, 32),
                            batch_size=batch_size,
                            border_mode="same")

model = ConvAutoEncoder(
    layers=[layer_0, layer_1, layer_2,
            MaxPoolingSameSize((32, 32)), layer_3])
#model=ConvAutoEncoder(layers=[layer_0, layer_3]);

out = model.fprop(images, corruption_level=None)
cost = mean_square_cost(out[-1], images)
#+L2_regularization(model.params, 0.005);

updates = gd_updates(cost=cost,
                     params=model.params,
                     method="sgd",
                     learning_rate=0.001,
                     momentum=0.9)

train = theano.function(
    inputs=[idx],
    outputs=[cost],
    updates=updates,
    givens={X: train_set_x[idx * batch_size:(idx + 1) * batch_size]})

print "[MESSAGE] The model is built"

epoch = 0
while (epoch < n_epochs):
    epoch = epoch + 1
    c = []
X=T.matrix("data");
y=T.ivector("label");
idx=T.lscalar();

layers=[ReLULayer(in_dim=784, out_dim=50)];

for i in xrange(20):
    layers.append(HighwayReLULayer(in_dim=50));
    
layers.append(SoftmaxLayer(in_dim=50, out_dim=10));

model=FeedForward(layers=layers);
out=model.fprop(X);
cost=categorical_cross_entropy_cost(out[-1], y);
updates=gd_updates(cost=cost, params=model.params, method="sgd", learning_rate=0.01, momentum=0.9);


train=theano.function(inputs=[idx],
                      outputs=cost,
                      updates=updates,
                      givens={X: train_set_x[idx * batch_size: (idx + 1) * batch_size],
                              y: train_set_y[idx * batch_size: (idx + 1) * batch_size]});

test=theano.function(inputs=[idx],
                     outputs=model.layers[-1].error(out[-1], y),
                     givens={X: test_set_x[idx * batch_size: (idx + 1) * batch_size],
                             y: test_set_y[idx * batch_size: (idx + 1) * batch_size]});
                             
print "[MESSAGE] The model is built"
Esempio n. 3
0
def dconvae_experiment(nkerns,
                       n_epochs,
                       batch_size,
                       start_noise,
                       noise_step):
    """ Setup dConvAE experiment
    
    This experiment is setting up for testing
    multi level noises on dConvAE using CIFAR-10
    
    Parameters
    ----------
    nkerns : int
        number of filters
    n_epochs : int
        number of training epochs
    batch_size : int
        size of each training batch
    start_noise : float
        start level of noise
    noise_step : float
        incremental steps of noises
    """
    
#     Xtr, Ytr, _, _=ds.load_CIFAR10("../data/CIFAR10");
# 
#     Xtr=np.mean(Xtr, 3);
#     # Xte=np.mean(Xte, 3);
#     Xtrain=Xtr.reshape(Xtr.shape[0], Xtr.shape[1]*Xtr.shape[2])/255.0;
#     # Xtest=Xte.reshape(Xte.shape[0], Xte.shape[1]*Xte.shape[2])/255.0;
# 
#     train_set_x, _=ds.shared_dataset((Xtrain, Ytr));
#     # test_set_x, test_set_y=ds.shared_dataset((Xtest, Yte));
# 
#     n_train_batches=train_set_x.get_value(borrow=True).shape[0]/batch_size;
#     # n_test_batches=test_set_x.get_value(borrow=True).shape[0]/batch_size;

    datasets=ds.load_mnist("../data/mnist.pkl.gz");
    train_set_x, train_set_y = datasets[0];
    valid_set_x, valid_set_y = datasets[1];
    test_set_x, test_set_y = datasets[2];

    n_train_batches=train_set_x.get_value(borrow=True).shape[0]/batch_size;
    n_valid_batches=valid_set_x.get_value(borrow=True).shape[0]/batch_size;
    n_test_batches=test_set_x.get_value(borrow=True).shape[0]/batch_size;

    print "[MESSAGE] The data is loaded"
    
    X=T.matrix("data");
    # y=T.ivector("label");
    idx=T.lscalar();
    corruption_level=T.fscalar();
    
    images=X.reshape((batch_size, 1, 28, 28))
    layer_0=ReLUConvLayer(filter_size=(5,5),
                          num_filters=nkerns,
                          num_channels=1,
                          fm_size=(28,28),
                          batch_size=batch_size,
                          border_mode="same");
                                                  
    layer_1=IdentityConvLayer(filter_size=(11,11),
                              num_filters=1,
                              num_channels=nkerns,
                              fm_size=(28,28),
                              batch_size=batch_size,
                              border_mode="same");
                         
    model=ConvAutoEncoder(layers=[layer_0, MaxPoolingSameSize((28,28)), layer_1]);

    out=model.fprop(images, corruption_level=corruption_level);
    cost=mean_square_cost(out[-1], images)#+L2_regularization(model.params, 0.005);
    
    updates=gd_updates(cost=cost, params=model.params, method="sgd", learning_rate=0.001, momentum=0.975);
    
    train=theano.function(inputs=[idx, corruption_level],
                      outputs=[cost],
                      updates=updates,
                      givens={X: train_set_x[idx * batch_size: (idx + 1) * batch_size]});
                      
    print "[MESSAGE] The model is built"

    epoch = 0;
    min_cost=None;
    corr=np.random.uniform(low=start_noise, high=start_noise+0.1, size=1).astype("float32");
    corr_best=corr[0];
    max_iter=0;
    while (epoch < n_epochs):
        epoch = epoch + 1;
        c = []
        for batch_index in xrange(n_train_batches):
            train_cost=train(batch_index, corr_best)
            c.append(train_cost);
            
        if min_cost==None:
            min_cost=np.mean(c);
        else:
            if (np.mean(c)<min_cost*0.5) or (max_iter>=30):
                min_cost=np.mean(c);
                corr=np.random.uniform(low=corr_best, high=corr_best+noise_step, size=1).astype("float32");
                corr_best=corr[0]
                max_iter=0;
            else:
                max_iter+=1;
            
        print 'Training epoch %d, cost %f, min cost %f, curr best %f, curr iter %d' % (epoch, np.mean(c), min_cost, corr_best, max_iter);
    
    filters=model.layers[-1].filters.get_value(borrow=True);

    for i in xrange(nkerns):
        image_adr="../data/dConvAE_multi_level/dConvAE_multi_level_%d.eps" % (i);
        plt.imshow(filters[0, i, :, :], cmap = plt.get_cmap('gray'), interpolation='nearest');
        plt.axis('off');
        plt.savefig(image_adr , bbox_inches='tight', pad_inches=0);
Esempio n. 4
0
layer_3 = SoftmaxLayer(in_dim=200, out_dim=10)

#dropout=multi_dropout([(batch_size, 1, 28, 28), None, (batch_size, 50, 11, 11), None, None, None, None], prob=0.5);
dropout = multi_dropout([(batch_size, 1, 28, 28), None,
                         (batch_size, 50, 11, 11), None, None, None, None],
                        prob=0.5)

model = FeedForward(
    layers=[layer_0, pool_0, layer_1, pool_1, flattener, layer_2, layer_3],
    dropout=dropout)

out = model.fprop(images)
cost = categorical_cross_entropy_cost(out[-1], y) + L2_regularization(
    model.params, 0.01)
updates = gd_updates(cost=cost,
                     params=model.params,
                     method="sgd",
                     learning_rate=0.1)

train = theano.function(
    inputs=[idx],
    outputs=cost,
    updates=updates,
    givens={
        X: train_set_x[idx * batch_size:(idx + 1) * batch_size],
        y: train_set_y[idx * batch_size:(idx + 1) * batch_size]
    })

test = theano.function(inputs=[idx],
                       outputs=model.layers[-1].error(out[-1], y),
                       givens={
                           X:
layer_0_de = IdentityConvLayer(filter_size=(7, 7),
                               num_filters=1,
                               num_channels=64,
                               fm_size=(32, 32),
                               batch_size=batch_size,
                               border_mode="same")

model = ConvAutoEncoder(layers=[layer_0_en, layer_0_de])

out = model.fprop(images, corruption_level=corruption_level)
cost = mean_square_cost(out[-1],
                        images)  #+L2_regularization(model.params, 0.005);

updates = gd_updates(cost=cost,
                     params=model.params,
                     method="sgd",
                     learning_rate=0.01,
                     momentum=0.9)

train = theano.function(
    inputs=[idx, corruption_level],
    outputs=[cost],
    updates=updates,
    givens={X: train_set_x[idx * batch_size:(idx + 1) * batch_size]})

print "[MESSAGE] The Layer 0 model is built"

epoch = 0
corr = np.random.uniform(low=0.1, high=0.2, size=1).astype("float32")
min_cost = None
corr_best = corr[0]
Esempio n. 6
0
print "[MESSAGE] The data is loaded"

X = T.matrix("data")
idx = T.lscalar()
noise = theano.shared(np.asarray(np.random.normal(scale=0.1,
                                                  size=(batch_size, 784)),
                                 dtype="float32"),
                      borrow=True)

#corrupted=corrupt_input(X, corruption_level=noise, noise_type="gaussian");
corrupted = X + noise

cost = binary_cross_entropy_cost(corrupted, X)

updates = gd_updates(cost, [noise], method="sgd", learning_rate=0.001)

train = theano.function(
    inputs=[idx],
    outputs=[cost],
    updates=updates,
    givens={X: train_set_x[idx * batch_size:(idx + 1) * batch_size]})

epoch = 0
while (epoch < n_epochs):
    epoch = epoch + 1
    c = []

    for batch_index in xrange(n_train_batches):
        train_cost = train(batch_index)
        c.append(train_cost)
Esempio n. 7
0
ep_idx=T.lscalar();
corruption_level=T.fscalar();

encode_layer=SigmoidLayer(in_dim=784,
                          out_dim=500);
                       
decode_layer=SigmoidLayer(in_dim=500,
                          out_dim=784);
                          
model=AutoEncoder(layers=[encode_layer, decode_layer]);

#out=model.fprop(X, corruption_level=corruption_level, noise_type="gaussian");
out=model.fprop(X, corruption_level=corruption_level);
cost=binary_cross_entropy_cost(out[-1], X);

updates=gd_updates(cost=cost, params=model.params, method="sgd", learning_rate=0.1);

train=theano.function(inputs=[idx, corruption_level],
                      outputs=[cost],
                      updates=updates,
                      givens={X: train_set_x[idx * batch_size: (idx + 1) * batch_size]});
                      
print "[MESSAGE] The model is built"

epoch = 0;
min_cost=None;
corr=np.random.uniform(low=0.2, high=0.3, size=1).astype("float32");
corr_best=corr[0]
while (epoch < n_epochs):
    epoch = epoch + 1;
    c = []
Esempio n. 8
0
n_test_batches=test_set_x.get_value(borrow=True).shape[0]/batch_size;

print "[MESSAGE] The data is loaded"

X=T.matrix("data");
idx=T.lscalar();
noise=theano.shared(np.asarray(np.random.normal(scale=0.1, size=(batch_size, 784)),
                               dtype="float32"),
                    borrow=True);
                    
#corrupted=corrupt_input(X, corruption_level=noise, noise_type="gaussian");
corrupted=X+noise;

cost=binary_cross_entropy_cost(corrupted, X);

updates=gd_updates(cost, [noise], method="sgd", learning_rate=0.001);

train=theano.function(inputs=[idx],
                      outputs=[cost],
                      updates=updates,
                      givens={X: train_set_x[idx * batch_size: (idx + 1) * batch_size]});
                      
epoch = 0;
while (epoch < n_epochs):
    epoch = epoch + 1;
    c = [];
    
    for batch_index in xrange(n_train_batches):
        train_cost=train(batch_index);
        c.append(train_cost);
        #co.append(curr_corr);
idx=T.lscalar();

layers=[ReLULayer(in_dim=6, out_dim=30)];

for i in xrange(20):
    layers.append(HighwayReLULayer(in_dim=30));
    
layers.append(ReLULayer(in_dim=30, out_dim=1));

model=FeedForward(layers=layers);
out=model.fprop(X);
cost=mean_squared_cost(out[-1], y)+L2_regularization(model.params, 0.01);
#cost=binary_cross_entropy_cost(out[-1], y)+L2_regularization(model.params, 0.01);

#updates=gd_updates(cost=cost, params=model.params, method="adagrad", learning_rate=0.01);
updates=gd_updates(cost=cost, params=model.params, method="rmsprop", learning_rate=0.001);

train=theano.function(inputs=[idx],
                      outputs=cost,
                      updates=updates,
                      givens={X: X_train[idx * batch_size: (idx + 1) * batch_size],
                              y: y_train[idx * batch_size: (idx + 1) * batch_size]});

test=theano.function(inputs=[idx],
                     outputs=cost,
                     givens={X: X_test[idx * batch_size: (idx + 1) * batch_size],
                             y: y_test[idx * batch_size: (idx + 1) * batch_size]});

print "[MESSAGE] The model is built"

test_record=np.zeros((n_epochs, 1));