print "[MESSAGE] The data is loaded" X=T.matrix("data"); y=T.ivector("label"); idx=T.lscalar(); layers=[ReLULayer(in_dim=784, out_dim=50)]; for i in xrange(20): layers.append(HighwayReLULayer(in_dim=50)); layers.append(SoftmaxLayer(in_dim=50, out_dim=10)); model=FeedForward(layers=layers); out=model.fprop(X); cost=categorical_cross_entropy_cost(out[-1], y); updates=gd_updates(cost=cost, params=model.params, method="sgd", learning_rate=0.01, momentum=0.9); train=theano.function(inputs=[idx], outputs=cost, updates=updates, givens={X: train_set_x[idx * batch_size: (idx + 1) * batch_size], y: train_set_y[idx * batch_size: (idx + 1) * batch_size]}); test=theano.function(inputs=[idx], outputs=model.layers[-1].error(out[-1], y), givens={X: test_set_x[idx * batch_size: (idx + 1) * batch_size], y: test_set_y[idx * batch_size: (idx + 1) * batch_size]}); print "[MESSAGE] The model is built"
print "[MESSAGE] The data is loaded" X = T.matrix("data") y = T.ivector("label") idx = T.lscalar() layers = [ReLULayer(in_dim=784, out_dim=50)] for i in xrange(20): layers.append(HighwayReLULayer(in_dim=50)) layers.append(SoftmaxLayer(in_dim=50, out_dim=10)) model = FeedForward(layers=layers) out = model.fprop(X) cost = categorical_cross_entropy_cost(out[-1], y) updates = gd_updates(cost=cost, params=model.params, method="sgd", learning_rate=0.01, momentum=0.9) train = theano.function( inputs=[idx], outputs=cost, updates=updates, givens={ X: train_set_x[idx * batch_size:(idx + 1) * batch_size], y: train_set_y[idx * batch_size:(idx + 1) * batch_size] })
layer_2 = ReLULayer(in_dim=320, out_dim=200) layer_3 = SoftmaxLayer(in_dim=200, out_dim=10) #dropout=multi_dropout([(batch_size, 1, 28, 28), None, (batch_size, 50, 11, 11), None, None, None, None], prob=0.5); dropout = multi_dropout([(batch_size, 1, 28, 28), None, (batch_size, 50, 11, 11), None, None, None, None], prob=0.5) model = FeedForward( layers=[layer_0, pool_0, layer_1, pool_1, flattener, layer_2, layer_3], dropout=dropout) out = model.fprop(images) cost = categorical_cross_entropy_cost(out[-1], y) + L2_regularization( model.params, 0.01) updates = gd_updates(cost=cost, params=model.params, method="sgd", learning_rate=0.1) train = theano.function( inputs=[idx], outputs=cost, updates=updates, givens={ X: train_set_x[idx * batch_size:(idx + 1) * batch_size], y: train_set_y[idx * batch_size:(idx + 1) * batch_size] })
print "[MESSAGE] The Lyer 1 model is trained" ################################## BUILD SUPERVISED MODEL ####################################### pool_0=MaxPooling(pool_size=(2,2)); pool_1=MaxPooling(pool_size=(2,2)); flattener=Flattener(); layer_2=ReLULayer(in_dim=32*64, out_dim=800); layer_3=SoftmaxLayer(in_dim=800, out_dim=10); model_sup=FeedForward(layers=[layer_0_en, pool_0, layer_1_en, pool_1, flattener, layer_2, layer_3]); out_sup=model_sup.fprop(images); cost_sup=categorical_cross_entropy_cost(out_sup[-1], y); updates=gd_updates(cost=cost_sup, params=model_sup.params, method="sgd", learning_rate=0.01, momentum=0.9); train_sup=theano.function(inputs=[idx], outputs=cost_sup, updates=updates, givens={X: train_set_x[idx * batch_size: (idx + 1) * batch_size], y: train_set_y[idx * batch_size: (idx + 1) * batch_size]}); test_sup=theano.function(inputs=[idx], outputs=model_sup.layers[-1].error(out_sup[-1], y), givens={X: test_set_x[idx * batch_size: (idx + 1) * batch_size], y: test_set_y[idx * batch_size: (idx + 1) * batch_size]}); print "[MESSAGE] The supervised model is built"
print "[MESSAGE] The Lyer 1 model is trained" ################################## BUILD SUPERVISED MODEL ####################################### pool_0 = MaxPooling(pool_size=(2, 2)) pool_1 = MaxPooling(pool_size=(2, 2)) flattener = Flattener() layer_2 = ReLULayer(in_dim=32 * 64, out_dim=800) layer_3 = SoftmaxLayer(in_dim=800, out_dim=10) model_sup = FeedForward(layers=[ layer_0_en, pool_0, layer_1_en, pool_1, flattener, layer_2, layer_3 ]) out_sup = model_sup.fprop(images) cost_sup = categorical_cross_entropy_cost(out_sup[-1], y) updates = gd_updates(cost=cost_sup, params=model_sup.params, method="sgd", learning_rate=0.01, momentum=0.9) train_sup = theano.function( inputs=[idx], outputs=cost_sup, updates=updates, givens={ X: train_set_x[idx * batch_size:(idx + 1) * batch_size], y: train_set_y[idx * batch_size:(idx + 1) * batch_size] })
flattener=Flattener(); layer_2=ReLULayer(in_dim=320, out_dim=200); layer_3=SoftmaxLayer(in_dim=200, out_dim=10); #dropout=multi_dropout([(batch_size, 1, 28, 28), None, (batch_size, 50, 11, 11), None, None, None, None], prob=0.5); dropout=multi_dropout([(batch_size, 1, 28, 28), None, (batch_size, 50, 11, 11), None, None, None, None], prob=0.5); model=FeedForward(layers=[layer_0, pool_0, layer_1, pool_1, flattener, layer_2, layer_3], dropout=dropout); out=model.fprop(images); cost=categorical_cross_entropy_cost(out[-1], y)+L2_regularization(model.params, 0.01); updates=gd_updates(cost=cost, params=model.params, method="sgd", learning_rate=0.1); train=theano.function(inputs=[idx], outputs=cost, updates=updates, givens={X: train_set_x[idx * batch_size: (idx + 1) * batch_size], y: train_set_y[idx * batch_size: (idx + 1) * batch_size]}); test=theano.function(inputs=[idx], outputs=model.layers[-1].error(out[-1], y), givens={X: test_set_x[idx * batch_size: (idx + 1) * batch_size], y: test_set_y[idx * batch_size: (idx + 1) * batch_size]}); print "[MESSAGE] The model is built"