print "[MESSAGE] The data is loaded"
X=T.matrix("data");
y=T.ivector("label");
idx=T.lscalar();
layers=[ReLULayer(in_dim=784, out_dim=50)];
for i in xrange(20):
layers.append(HighwayReLULayer(in_dim=50));
layers.append(SoftmaxLayer(in_dim=50, out_dim=10));
model=FeedForward(layers=layers);
out=model.fprop(X);
cost=categorical_cross_entropy_cost(out[-1], y);
updates=gd_updates(cost=cost, params=model.params, method="sgd", learning_rate=0.01, momentum=0.9);
train=theano.function(inputs=[idx],
outputs=cost,
updates=updates,
givens={X: train_set_x[idx * batch_size: (idx + 1) * batch_size],
y: train_set_y[idx * batch_size: (idx + 1) * batch_size]});
test=theano.function(inputs=[idx],
outputs=model.layers[-1].error(out[-1], y),
givens={X: test_set_x[idx * batch_size: (idx + 1) * batch_size],
y: test_set_y[idx * batch_size: (idx + 1) * batch_size]});
print "[MESSAGE] The model is built"
print "[MESSAGE] The data is loaded"
X = T.matrix("data")
y = T.ivector("label")
idx = T.lscalar()
layers = [ReLULayer(in_dim=784, out_dim=50)]
for i in xrange(20):
layers.append(HighwayReLULayer(in_dim=50))
layers.append(SoftmaxLayer(in_dim=50, out_dim=10))
model = FeedForward(layers=layers)
out = model.fprop(X)
cost = categorical_cross_entropy_cost(out[-1], y)
updates = gd_updates(cost=cost,
params=model.params,
method="sgd",
learning_rate=0.01,
momentum=0.9)
train = theano.function(
inputs=[idx],
outputs=cost,
updates=updates,
givens={
X: train_set_x[idx * batch_size:(idx + 1) * batch_size],
y: train_set_y[idx * batch_size:(idx + 1) * batch_size]
})
layer_2 = ReLULayer(in_dim=320, out_dim=200)
layer_3 = SoftmaxLayer(in_dim=200, out_dim=10)
#dropout=multi_dropout([(batch_size, 1, 28, 28), None, (batch_size, 50, 11, 11), None, None, None, None], prob=0.5);
dropout = multi_dropout([(batch_size, 1, 28, 28), None,
(batch_size, 50, 11, 11), None, None, None, None],
prob=0.5)
model = FeedForward(
layers=[layer_0, pool_0, layer_1, pool_1, flattener, layer_2, layer_3],
dropout=dropout)
out = model.fprop(images)
cost = categorical_cross_entropy_cost(out[-1], y) + L2_regularization(
model.params, 0.01)
updates = gd_updates(cost=cost,
params=model.params,
method="sgd",
learning_rate=0.1)
train = theano.function(
inputs=[idx],
outputs=cost,
updates=updates,
givens={
X: train_set_x[idx * batch_size:(idx + 1) * batch_size],
y: train_set_y[idx * batch_size:(idx + 1) * batch_size]
})
print "[MESSAGE] The Lyer 1 model is trained"
################################## BUILD SUPERVISED MODEL #######################################
pool_0=MaxPooling(pool_size=(2,2));
pool_1=MaxPooling(pool_size=(2,2));
flattener=Flattener();
layer_2=ReLULayer(in_dim=32*64,
out_dim=800);
layer_3=SoftmaxLayer(in_dim=800,
out_dim=10);
model_sup=FeedForward(layers=[layer_0_en, pool_0, layer_1_en, pool_1, flattener, layer_2, layer_3]);
out_sup=model_sup.fprop(images);
cost_sup=categorical_cross_entropy_cost(out_sup[-1], y);
updates=gd_updates(cost=cost_sup, params=model_sup.params, method="sgd", learning_rate=0.01, momentum=0.9);
train_sup=theano.function(inputs=[idx],
outputs=cost_sup,
updates=updates,
givens={X: train_set_x[idx * batch_size: (idx + 1) * batch_size],
y: train_set_y[idx * batch_size: (idx + 1) * batch_size]});
test_sup=theano.function(inputs=[idx],
outputs=model_sup.layers[-1].error(out_sup[-1], y),
givens={X: test_set_x[idx * batch_size: (idx + 1) * batch_size],
y: test_set_y[idx * batch_size: (idx + 1) * batch_size]});
print "[MESSAGE] The supervised model is built"
print "[MESSAGE] The Lyer 1 model is trained"
################################## BUILD SUPERVISED MODEL #######################################
pool_0 = MaxPooling(pool_size=(2, 2))
pool_1 = MaxPooling(pool_size=(2, 2))
flattener = Flattener()
layer_2 = ReLULayer(in_dim=32 * 64, out_dim=800)
layer_3 = SoftmaxLayer(in_dim=800, out_dim=10)
model_sup = FeedForward(layers=[
layer_0_en, pool_0, layer_1_en, pool_1, flattener, layer_2, layer_3
])
out_sup = model_sup.fprop(images)
cost_sup = categorical_cross_entropy_cost(out_sup[-1], y)
updates = gd_updates(cost=cost_sup,
params=model_sup.params,
method="sgd",
learning_rate=0.01,
momentum=0.9)
train_sup = theano.function(
inputs=[idx],
outputs=cost_sup,
updates=updates,
givens={
X: train_set_x[idx * batch_size:(idx + 1) * batch_size],
y: train_set_y[idx * batch_size:(idx + 1) * batch_size]
})
flattener=Flattener();
layer_2=ReLULayer(in_dim=320,
out_dim=200);
layer_3=SoftmaxLayer(in_dim=200,
out_dim=10);
#dropout=multi_dropout([(batch_size, 1, 28, 28), None, (batch_size, 50, 11, 11), None, None, None, None], prob=0.5);
dropout=multi_dropout([(batch_size, 1, 28, 28), None, (batch_size, 50, 11, 11), None, None, None, None], prob=0.5);
model=FeedForward(layers=[layer_0, pool_0, layer_1, pool_1, flattener, layer_2, layer_3],
dropout=dropout);
out=model.fprop(images);
cost=categorical_cross_entropy_cost(out[-1], y)+L2_regularization(model.params, 0.01);
updates=gd_updates(cost=cost, params=model.params, method="sgd", learning_rate=0.1);
train=theano.function(inputs=[idx],
outputs=cost,
updates=updates,
givens={X: train_set_x[idx * batch_size: (idx + 1) * batch_size],
y: train_set_y[idx * batch_size: (idx + 1) * batch_size]});
test=theano.function(inputs=[idx],
outputs=model.layers[-1].error(out[-1], y),
givens={X: test_set_x[idx * batch_size: (idx + 1) * batch_size],
y: test_set_y[idx * batch_size: (idx + 1) * batch_size]});
print "[MESSAGE] The model is built"
