def train(self, X, y_train, X_test, ids_test, y_test, outfile, is_valid):
X = np.array(X)
encoder = LabelEncoder()
y = encoder.fit_transform(y_train).astype(np.int32)
num_classes = len(encoder.classes_)
num_features = X.shape[1]
layers0 = [('input', InputLayer),
('dense1', DenseLayer),
('dropout1', DropoutLayer),
('dense2', DenseLayer),
('dropout2', DropoutLayer),
('output', DenseLayer)]
net0 = NeuralNet(layers=layers0,
input_shape=(None, num_features),
dense1_num_units=3500,
dropout1_p=0.4,
dense2_num_units=2300,
dropout2_p=0.5,
output_num_units=num_classes,
output_nonlinearity=softmax,
#update=nesterov_momentum,
update=adagrad,
update_learning_rate=0.01,
#update_momentum=0.9,
#objective_loss_function=softmax,
objective_loss_function=categorical_crossentropy,
eval_size=0.2,
verbose=1,
max_epochs=20)
net0.fit(X, y)
X_test = np.array(X_test)
self.make_submission(net0, X_test, ids_test, encoder)
def loadNet(netName):
if os.path.exists(netName):
net = pickle.load(open(netName, "rb"))
else:
net = NeuralNet(
layers=[ # three layers: one hidden layer
('input', layers.InputLayer),
('hidden', layers.DenseLayer),
('output', layers.DenseLayer),
],
# layer parameters:
input_shape=(None, 9216), # 96x96 input pixels per batch
hidden_num_units=100, # number of units in hidden layer
output_nonlinearity=None, # output layer uses identity function
output_num_units=30, # 30 target values
# optimization method:
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
regression=True, # flag to indicate we're dealing with regression problem
max_epochs=400, # we want to train this many epochs
verbose=1,
)
X, y = load()
net.fit(X, y)
print("X.shape == {}; X.min == {:.3f}; X.max == {:.3f}".format(X.shape, X.min(), X.max()))
print("y.shape == {}; y.min == {:.3f}; y.max == {:.3f}".format(y.shape, y.min(), y.max()))
pickle.dump(net, open(netName, 'wb'), -1)
return net
def fit_model(train_x, y, test_x):
"""Feed forward neural network for kaggle digit recognizer competition.
Intentionally limit network size and optimization time (by choosing max_epochs = 15) to meet runtime restrictions
"""
print("\n\nRunning Convetional Net. Optimization progress below\n\n")
net1 = NeuralNet(
layers=[ #list the layers here
('input', layers.InputLayer),
('hidden1', layers.DenseLayer),
('output', layers.DenseLayer),
],
# layer parameters:
input_shape=(None, train_x.shape[1]),
hidden1_num_units=200, hidden1_nonlinearity=rectify, #params of first layer
output_nonlinearity=softmax, # softmax for classification problems
output_num_units=10, # 10 target values
# optimization method:
update=nesterov_momentum,
update_learning_rate=0.05,
update_momentum=0.7,
regression=False,
max_epochs=10, # Intentionally limited for execution speed
verbose=1,
)
net1.fit(train_x, y)
predictions = net1.predict(test_x)
return(predictions)
def lasagne_oneLayer_classifier(param, X, labels):
## initialize the NN
layers0 = [('input', InputLayer),
('dense0', DenseLayer),
('dropout', DropoutLayer),
('output', DenseLayer)]
net0 = NeuralNet(layers=layers0,
input_shape=(None, param['num_features']),
dense0_num_units=param['dense0_num_units'],
dropout_p=param['dropout_p'],
output_num_units=param['num_classes'],
output_nonlinearity=softmax,
update=nesterov_momentum,
update_learning_rate=param['update_learning_rate'],
update_momentum=param['update_momentum'],
eval_size=0.02,
verbose=1,
max_epochs=param['max_epochs'])
## fit the results
net0.fit(X, labels)
return net0
def train(x_train, y_train):
clf_nn = NeuralNet(
layers=[ # three layers: one hidden layer
('input', layers.InputLayer),
('hidden1', layers.DenseLayer),
('hidden2', layers.DenseLayer),
('output', layers.DenseLayer),
],
# layer parameters:
input_shape=(None, 2538), # 784 input pixels per batch
hidden1_num_units=100, # number of units in hidden layer
hidden2_num_units=100,
output_nonlinearity=nonlinearities.softmax, # output layer uses identity function
output_num_units=10, # 10 target values
# optimization method:
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
max_epochs=50, # we want to train this many epochs
verbose=1,
)
clf_nn.fit(x_train, y_train)
return clf_nn
def train():
weather = load_weather()
training = load_training()
X = assemble_X(training, weather)
print len(X[0])
mean, std = normalize(X)
y = assemble_y(training)
input_size = len(X[0])
learning_rate = theano.shared(np.float32(0.1))
net = NeuralNet(
layers=[
('input', InputLayer),
('hidden1', DenseLayer),
('dropout1', DropoutLayer),
('hidden2', DenseLayer),
('dropout2', DropoutLayer),
('output', DenseLayer),
],
# layer parameters:
input_shape=(None, input_size),
hidden1_num_units=325,
dropout1_p=0.4,
hidden2_num_units=325,
dropout2_p=0.4,
output_nonlinearity=sigmoid,
output_num_units=1,
# optimization method:
update=nesterov_momentum,
update_learning_rate=learning_rate,
update_momentum=0.9,
# Decay the learning rate
on_epoch_finished=[
AdjustVariable(learning_rate, target=0, half_life=1),
],
# This is silly, but we don't want a stratified K-Fold here
# To compensate we need to pass in the y_tensor_type and the loss.
regression=True,
y_tensor_type = T.imatrix,
objective_loss_function = binary_crossentropy,
max_epochs=85,
eval_size=0.1,
verbose=1,
)
X, y = shuffle(X, y, random_state=123)
net.fit(X, y)
_, X_valid, _, y_valid = net.train_test_split(X, y, net.eval_size)
probas = net.predict_proba(X_valid)[:,0]
print("ROC score", metrics.roc_auc_score(y_valid, probas))
return net, mean, std
def OptNN(d1, h1, d2, h2, d3, start, stop, max_epochs):
params2 = params.copy()
on_epoch = [AdjustVariable('update_learning_rate',
start = start, stop = stop),
AdjustVariable('update_momentum', start = .9, stop = .999)]
params2['dropout1_p'] = d1
params2['dropout2_p'] = d2
params2['dropout3_p'] = d3
params2['dropout4_p'] = d4
params2['hidden1_num_units'] = h1
params2['hidden2_num_units'] = h2
params2['hidden3_num_units'] = h3
params2['max_epochs'] = max_epochs
params2['on_epoch_finished'] = on_epoch
kcv = StratifiedKFold(Y, 5, shuffle = True)
res = np.empty((len(Y), len(np.unique(Y)))); i = 1
CVScores = []
for train_idx, valid_idx in kcv:
logger.info("Running fold %d...", i); i += 1
net = NeuralNet(**params2)
net.set_params(eval_size = None)
net.fit(X[train_idx], Y[train_idx])
res[valid_idx, :] = net.predict_proba(X[valid_idx])
CVScores.append(log_loss(Y[valid_idx], res[valid_idx]))
return -np.mean(CVScores)
def trainNet(X, Y, ln, loadFile = ""):
net1 = NeuralNet(
layers=[ # four layers: two hidden layers
('input', layers.InputLayer),
('hidden', layers.DenseLayer),
('hidden1', layers.DenseLayer),
('output', layers.DenseLayer),
],
# layer parameters: Best 400 400
input_shape=(None, numInputs), # 31 inputs
hidden_num_units=400, # number of units in hidden layer
hidden1_num_units=400,
hidden_nonlinearity=lasagne.nonlinearities.sigmoid,
hidden1_nonlinearity=lasagne.nonlinearities.sigmoid,
output_nonlinearity=None, # output layer uses identity function
output_num_units=numOutputs, # 4 outputs
# optimization method:
update=nesterov_momentum,
update_learning_rate=ln,
update_momentum=0.9,
regression=True, # flag to indicate we're dealing with regression problem
max_epochs=1500, # we want to train this many epochs
verbose=1,
)
#if (loadFile != ""):
#net1.load_params_from(loadFile)
net1.max_epochs = 10
net1.update_learning_rate = ln;
net1.fit(X, Y) # This thing try to do the fit itself
return net1
def fit(self,tr,add_feat_tr):
## if trend exists, remove trend
if self.trend ==1:
trend = self.est_trend(tr)
tr = tr-np.asarray(trend)
layers0=[
## 2 layers with one hidden layer
(InputLayer, {'shape': (None,8,self.window_length)}),
(DenseLayer, {'num_units': 8*self.window_length}),
(DropoutLayer, {'p':0.3}),
(DenseLayer, {'num_units': 8*self.window_length/3}),
## the output layer
(DenseLayer, {'num_units': 1, 'nonlinearity': None}),
]
feats = build_feat(tr, add_feat_tr, window_length=self.window_length)
print feats.shape
feat_target = get_target(tr,window_length=self.window_length)
print feat_target.shape
net0 = NeuralNet(
layers=layers0,
max_epochs=400,
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
verbose=1,
regression=True,
)
net0.fit(feats[:-1],feat_target)
return net0,feats,feat_target
def neural_network(x_train, y_train):
X, y, encoder, scaler = load_train_data(x_train, y_train)
num_classes = len(encoder.classes_)
num_features = X.shape[1]
layers0 = [
("input", InputLayer),
("dropoutf", DropoutLayer),
("dense0", DenseLayer),
("dropout", DropoutLayer),
("dense1", DenseLayer),
("dropout2", DropoutLayer),
("output", DenseLayer),
]
net0 = NeuralNet(
layers=layers0,
input_shape=(None, num_features),
dropoutf_p=0.15,
dense0_num_units=1000,
dropout_p=0.25,
dense1_num_units=500,
dropout2_p=0.25,
output_num_units=num_classes,
output_nonlinearity=softmax,
update=adagrad,
update_learning_rate=0.005,
eval_size=0.01,
verbose=1,
max_epochs=30,
)
net0.fit(X, y)
return (net0, scaler)
def fit(xTrain, yTrain, dense0_num=800, dropout_p=0.5, dense1_num=500, update_learning_rate=0.01,
update_momentum=0.9, test_ratio=0.2, max_epochs=20):
#update_momentum=0.9, test_ratio=0.2, max_epochs=20, train_fname='train.csv'):
#xTrain, yTrain, encoder, scaler = load_train_data(train_fname)
#xTest, ids = load_test_data('test.csv', scaler)
num_features = len(xTrain[0,:])
num_classes = 9
print num_features
layers0 = [('input', InputLayer),
('dense0', DenseLayer),
('dropout', DropoutLayer),
('dense1', DenseLayer),
('output', DenseLayer)]
clf = NeuralNet(layers=layers0,
input_shape=(None, num_features),
dense0_num_units=dense0_num,
dropout_p=dropout_p,
dense1_num_units=dense1_num,
output_num_units=num_classes,
output_nonlinearity=softmax,
update=nesterov_momentum,
update_learning_rate=update_learning_rate,
update_momentum=update_momentum,
eval_size=test_ratio,
verbose=1,
max_epochs=max_epochs)
clf.fit(xTrain, yTrain)
ll_train = metrics.log_loss(yTrain, clf.predict_proba(xTrain))
print ll_train
return clf
def NN(X,y):
net1 = NeuralNet(
layers=[ # three layers: one hidden layer
('input', layers.InputLayer),
('hidden', layers.DenseLayer),
('output', layers.DenseLayer),
],
# layer parameters:
input_shape=(None, 9216), # 96x96 input pixels per batch
hidden_num_units=100, # number of units in hidden layer
output_nonlinearity=None, # output layer uses identity function
output_num_units=30, # 30 target values
# optimization method:
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
regression=True, # flag to indicate we're dealing with regression problem
max_epochs=400, # we want to train this many epochs
verbose=1,
)
net1.fit(X, y)
def train_net(X, y):
net2 = NeuralNet(
layers=[
('input', layers.InputLayer),
('ncaa', NCAALayer),
('dropout1', layers.DropoutLayer),
('hidden', layers.DenseLayer),
('dropout2', layers.DropoutLayer),
('output', layers.DenseLayer),
],
input_shape = (None, num_features * 2),
ncaa_num_units = 128,
dropout1_p=0.2,
hidden_num_units=128,
dropout2_p=0.3,
output_nonlinearity=nonlinearities.sigmoid,
output_num_units=1,
update=nesterov_momentum,
update_learning_rate=theano.shared(float32(0.01)),
update_momentum=theano.shared(float32(0.9)),
regression=True, # flag to indicate we're dealing with regression problem
max_epochs=20, # we want to train this many epochs
verbose=1,
)
net2.fit(X, y)
return net2
def nn_example(data):
net1 = NeuralNet(
layers=[('input', layers.InputLayer),
('hidden', layers.DenseLayer),
('output', layers.DenseLayer),
],
# layer parameters:
input_shape=(None, 28*28),
hidden_num_units=100, # number of units in 'hidden' layer
output_nonlinearity=lasagne.nonlinearities.softmax,
output_num_units=10, # 10 target values for the digits 0, 1, 2, ..., 9
# optimization method:
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
max_epochs=10,
verbose=1,
)
# Train the network
net1.fit(data['X_train'], data['y_train'])
# Try the network on new data
print("Feature vector (100-110): %s" % data['X_test'][0][100:110])
print("Label: %s" % str(data['y_test'][0]))
print("Predicted: %s" % str(net1.predict([data['X_test'][0]])))
def gridsearch_alpha(self,learning_rate,index,params=None):
hidden_unit = ((index+1)*2)/3
self.l_in = ls.layers.InputLayer(shape=(None,n_input),input_var=None)
self.l_hidden = ls.layers.DenseLayer(self.l_in,num_units=15,nonlinearity=ls.nonlinearities.rectify)
self.network = l_out = ls.layers.DenseLayer(self.l_hidden,num_units=1)
list_results = np.array([learning_rate.shape[0]],dtype=np.float64)
for item in learning_rate:
#Init Neural net
net1 = NeuralNet(
layers=self.network,
# optimization method:
update=nesterov_momentum,
update_learning_rate=item,
update_momentum=0.9,
regression=True, # flag to indicate we're dealing with regression problem
max_epochs=800, # we want to train this many epochs
# verbose=1,
eval_size = 0.4
)
net1.fit(self.X_training,self.y_training)
self.pred = net1.predict(self.n_sample2)
name_file = "Params/saveNeuralNetwork_%s_%s.tdn" %(item,index)
net1.save_params_to(name_file)
score_nn = net1.score(self.n_sample2,self.n_test2)
list_results[item] = score_nn
print "index=%f,item=%f,score=%f"%(index,item,score_nn)
return list_results
def build_mlp(input_var=None):
net1 = NeuralNet(
layers=[ # three layers: one hidden layer
('input', layers.InputLayer),
('hidden1', layers.DenseLayer),
('hidden2', layers.DenseLayer),
('output', layers.DenseLayer),
],
# layer parameters:
input_shape=(None, 14, 2177), # 14 x 2177 input pixels per batch
hidden1_num_units=100, # number of units in hidden layer
hidden2_num_units=100,
output_nonlinearity=lasagne.nonlinearities.softmax, # output layer uses identity function
output_num_units=2, # 2 target values
# optimization method:
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
#regression=False, # flag to indicate we're dealing with regression problem
max_epochs=500, # we want to train this many epochs
verbose=1,
)
X, y = load_dataset()
y = np.asanyarray(y,np.int32)
print(X.shape)
print(y.shape)
net1.fit(X, y)
def train_network():
layers0 = [('input', InputLayer),
('dense0', DenseLayer),
('dropout0', DropoutLayer),
('dense1', DenseLayer),
('dropout1', DropoutLayer),
('dense2', DenseLayer),
('output', DenseLayer)]
es = EarlyStopping(patience=200)
net0 = NeuralNet(layers=layers0,
input_shape=(None, num_features),
dense0_num_units=256,
dropout0_p=0.5,
dense1_num_units=128,
dropout1_p=0.5,
dense2_num_units=64,
output_num_units=num_classes,
output_nonlinearity=softmax,
update=nesterov_momentum,
update_learning_rate=theano.shared(float32(0.01)),
update_momentum=theano.shared(float32(0.9)),
eval_size=0.2,
verbose=1,
max_epochs=1000,
on_epoch_finished=[
AdjustVariable('update_learning_rate', start=0.01, stop=0.0001),
AdjustVariable('update_momentum', start=0.9, stop=0.999),
es
])
net0.fit(X, y)
return (es.best_valid, net0)
class NN(object):
def __init__(self, input_size, hidden_1_size, hidden_2_size=None):
n_layers = [
('input', layers.InputLayer),
('hidden1', layers.DenseLayer),
('dropout1', layers.DropoutLayer)
]
if hidden_2_size is not None:
n_layers.extend(
[('hidden2', layers.DenseLayer), ('dropout2', layers.DropoutLayer)]
)
n_layers.append(('output', layers.DenseLayer))
self.model = NeuralNet(
layers=n_layers,
input_shape=(None, input_size),
hidden1_num_units=hidden_1_size, dropout1_p=0.5,
output_nonlinearity=tanh,
output_num_units=1,
regression=True,
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
eval_size=0.1,
on_epoch_finished=[
AdjustVariable('update_learning_rate', stop=0.0001, decrement=0.00001),
AdjustVariable('update_momentum', stop=0.999, increment=0.0001),
EarlyStopping(patience=100)
],
max_epochs=5000,
verbose=1
)
if hidden_2_size is not None:
self.model.__dict__['hidden2_num_units'] = hidden_2_size
self.model.__dict__['dropout2_p'] = 0.5
def train(self, X, Y):
self.model.fit(np.asarray(X, dtype=np.float32), np.asarray(Y, dtype=np.float32))
def predict_continuous(self, X_test):
return self.model.predict(np.asarray(X_test, dtype=np.float32))
def predict_classes(self, X_test):
Y_pred = self.predict_continuous(X_test)
# threshold the continuous values to get the classes
pos = Y_pred >= .33
neg = Y_pred <= -0.33
neu = np.logical_and(Y_pred < 0.33, Y_pred > -0.33)
Y_pred[pos] = 1
Y_pred[neg] = -1
Y_pred[neu] = 0
return Y_pred.reshape(-1)
def regr(X, Y): l = InputLayer(shape=(None, X.shape[1])) l = DenseLayer(l, num_units=X.shape[1], nonlinearity=tanh) #tanh, sigmoid # l = DropoutLayer(l, p=0.3, rescale=True) # previous: p=0.5 l = DenseLayer(l, num_units=1, nonlinearity=sigmoid) # l = DropoutLayer(l, p=0.3, rescale=True) # previous: p=0.5 net = NeuralNet(l, regression=True, update_learning_rate=0.01, verbose=1, max_epochs=700) net.fit(X, Y) print(net.score(X, Y)) return net
class RegressionNN(RegressionBase.RegressionBase):
def __init__(self, isTrain, isNN):
super(RegressionNN, self).__init__(isTrain, isNN)
# data preprocessing
#self.dataPreprocessing()
self.net1 = NeuralNet(
layers=[ # three layers: one hidden layer
('input', layers.InputLayer),
('hidden', layers.DenseLayer),
#('hidden2', layers.DenseLayer),
#('hidden3', layers.DenseLayer),
('output', layers.DenseLayer),
],
# layer parameters:
input_shape=(None, 13), # input dimension is 13
hidden_num_units=6, # number of units in hidden layer
#hidden2_num_units=8, # number of units in hidden layer
#hidden3_num_units=4, # number of units in hidden layer
output_nonlinearity=None, # output layer uses sigmoid function
output_num_units=1, # output dimension is 1
# obejctive function
objective_loss_function = lasagne.objectives.squared_error,
# optimization method:
update=lasagne.updates.nesterov_momentum,
update_learning_rate=0.002,
update_momentum=0.4,
# use 25% as validation
train_split=TrainSplit(eval_size=0.2),
regression=True, # flag to indicate we're dealing with regression problem
max_epochs=100, # we want to train this many epochs
verbose=0,
)
def dataPreprocessing(self):
# due to the observation, standization does not help the optimization.
# So do not use it!
#self.Standardization()
pass
def training(self):
# train the NN model
self.net1.fit(self.X_train, self.y_train)
def predict(self):
# predict the test data
self.y_pred = self.net1.predict(self.X_test)
# print MSE
mse = mean_squared_error(self.y_pred, self.y_test)
print "MSE: {}".format(mse)
def regr(X, Y): l = InputLayer(shape=(None, X.shape[1])) l = DenseLayer(l, num_units=Y.shape[1]+100, nonlinearity=tanh) # l = DropoutLayer(l, p=0.3, rescale=True) # previous: p=0.5 l = DenseLayer(l, num_units=Y.shape[1]+50, nonlinearity=tanh) # l = DropoutLayer(l, p=0.3, rescale=True) # previous: p=0.5 l = DenseLayer(l, num_units=Y.shape[1], nonlinearity=None) net = NeuralNet(l, regression=True, update_learning_rate=0.1, verbose=1) net.fit(X, Y) print(net.score(X, Y)) return net
def OptNN2(d0, d1,d2, d3, h1, h2, h3, me, ls, le):
h1, h2, h3 = int(h1), int(h2), int(h3);
me = int(me)
params = dict(
layers = [
('input', layers.InputLayer),
('dropout1', layers.DropoutLayer),
('hidden1', layers.DenseLayer),
('dropout2', layers.DropoutLayer),
('hidden2', layers.DenseLayer),
('dropout3', layers.DropoutLayer),
('hidden3', layers.DenseLayer),
('dropout4', layers.DropoutLayer),
('output', layers.DenseLayer),
],
input_shape = (None, 93),
dropout1_p = d0,
hidden1_num_units = h1,
dropout2_p = d1,
hidden2_num_units = h2,
dropout3_p = d2,
hidden3_num_units = h3,
dropout4_p = d3,
output_nonlinearity = softmax,
output_num_units = 9,
update = nesterov_momentum,
update_learning_rate = theano.shared(float32(l_start)),
update_momentum = theano.shared(float32(m_start)),
regression = False,
on_epoch_finished = [
AdjustVariable('update_learning_rate', start = ls,
stop = le, is_log = True),
AdjustVariable('update_momentum', start = m_start,
stop = m_stop, is_log = False),
],
max_epochs = me,
verbose = 1,
)
CVScores = []
res = np.empty((len(Y), len(np.unique(Y))))
kcv = StratifiedKFold(Y, 5, shuffle = True); i = 1
for train_idx, valid_idx in kcv:
logger.info("Running fold %d...", i); i += 1
net = NeuralNet(**params)
net.set_params(eval_size = None)
net.fit(X[train_idx], Y[train_idx])
res[valid_idx, :] = net.predict_proba(X[valid_idx])
CVScores.append(log_loss(Y[valid_idx], res[valid_idx]))
return -np.mean(CVScores)
def test_lasagne_functional_regression(boston):
from nolearn.lasagne import NeuralNet
X, y = boston
layer1 = InputLayer(shape=(128, 13))
layer2 = DenseLayer(layer1, num_units=100)
output = DenseLayer(layer2, num_units=1, nonlinearity=identity)
nn = NeuralNet(layers=output, update_learning_rate=0.01, update_momentum=0.1, regression=True, max_epochs=50)
nn.fit(X[:300], y[:300])
assert mean_absolute_error(nn.predict(X[300:]), y[300:]) < 3.0
def linreg(X, y):
net = NeuralNet(
layers=[
('input', InputLayer),
('output', DenseLayer)],
input_shape=(None, X.shape[1]),
output_num_units=2,
output_nonlinearity=identity,
regression=True,
update_learning_rate=1e-9,
update_momentum=0.9,
verbose=1)
net.fit(X, y)
def test_lasagne_functional_mnist(mnist):
# Run a full example on the mnist dataset
from nolearn.lasagne import NeuralNet
X, y = mnist
X_train, y_train = X[:60000], y[:60000]
X_test, y_test = X[60000:], y[60000:]
epochs = []
def on_epoch_finished(nn, train_history):
epochs[:] = train_history
if len(epochs) > 1:
raise StopIteration()
nn = NeuralNet(
layers=[
('input', InputLayer),
('hidden1', DenseLayer),
('dropout1', DropoutLayer),
('hidden2', DenseLayer),
('dropout2', DropoutLayer),
('output', DenseLayer),
],
input_shape=(None, 784),
output_num_units=10,
output_nonlinearity=softmax,
more_params=dict(
hidden1_num_units=512,
hidden2_num_units=512,
),
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
max_epochs=5,
on_epoch_finished=on_epoch_finished,
)
nn.fit(X_train, y_train)
assert len(epochs) == 2
assert epochs[0]['valid_accuracy'] > 0.85
assert epochs[1]['valid_accuracy'] > epochs[0]['valid_accuracy']
assert sorted(epochs[0].keys()) == [
'epoch', 'train_loss', 'valid_accuracy', 'valid_loss',
]
y_pred = nn.predict(X_test)
assert accuracy_score(y_pred, y_test) > 0.85
class network(object):
def __init__(self,X_train, Y_train):
#self.__hidden=0
self.__hidden=int(math.ceil((2*(X_train.shape[1]+ 1))/3))
self.net= NeuralNet(
layers=[
('input', layers.InputLayer),
('hidden', layers.DenseLayer),
('output', layers.DenseLayer)
],
input_shape=( None, X_train.shape[1] ),
hidden_num_units=self.__hidden,
#hidden_nonlinearity=nonlinearities.tanh,
output_nonlinearity=None,
batch_iterator_train=BatchIterator(batch_size=256),
output_num_units=1,
on_epoch_finished=[EarlyStopping(patience=50)],
update=momentum,
update_learning_rate=theano.shared(np.float32(0.03)),
update_momentum=theano.shared(np.float32(0.8)),
regression=True,
max_epochs=1000,
verbose=1,
)
self.net.fit(X_train,Y_train)
def predict(self,X):
return self.net.predict(X)
def showMetrics(self):
train_loss = np.array([i["train_loss"] for i in self.net.train_history_])
valid_loss = np.array([i["valid_loss"] for i in self.net.train_history_])
pyplot.plot(train_loss, linewidth=3, label="training")
pyplot.plot(valid_loss, linewidth=3, label="validation")
pyplot.grid()
pyplot.legend()
pyplot.xlabel("epoch")
pyplot.ylabel("loss")
# pyplot.ylim(1e-3, 1e-2)
pyplot.yscale("log")
pyplot.show()
def saveNet(self,fname):
self.net.save_params_to(fname)
def loadNet(self,fname):
self.net.load_params_from(fname)
def nnet(pipe):
pipe.features = pipe.features.astype(np.float32)
pipe.labels = pipe.labels.astype(np.int32)
pipe.features = StandardScaler().fit_transform(pipe.features)
X_train, X_test, y_train, y_test = train_test_split(pipe.features, pipe.labels)
nnet = NeuralNet(
# Specify the layers
layers=[('input', layers.InputLayer),
('hidden1', layers.DenseLayer),
('hidden2', layers.DenseLayer),
('hidden3', layers.DenseLayer),
('output', layers.DenseLayer)],
# Input Layer
input_shape=(None, pipe.features.shape[1]),
# Hidden Layer 1
hidden1_num_units=512,
hidden1_nonlinearity=rectify,
# Hidden Layer 2
hidden2_num_units=512,
hidden2_nonlinearity=rectify,
# # Hidden Layer 3
hidden3_num_units=512,
hidden3_nonlinearity=rectify,
# Output Layer
output_num_units=2,
output_nonlinearity=softmax,
# Optimization
update=nesterov_momentum,
update_learning_rate=0.001,
update_momentum=0.3,
max_epochs=30,
# Others,
regression=False,
verbose=1,
)
nnet.fit(X_train, y_train)
y_predict = nnet.predict(X_test)
print "precision for nnet:", precision_score(y_test, y_predict)
print "recall for nnet:", recall_score(y_test, y_predict)
print "f1 for nnet:", f1_score(y_test, y_predict, average='weighted')
pickle.dump( nnet, open( "model.pkl", "wb" ), protocol = cPickle.HIGHEST_PROTOCOL)
def mlp(X, y):
net = NeuralNet(
layers=[
('input', InputLayer),
('hidden', DenseLayer),
('output', DenseLayer)],
input_shape=(None, X.shape[1]),
hidden_num_units=5,
hidden_nonlinearity=sigmoid,
output_num_units=2,
output_nonlinearity=sigmoid,
regression=True,
update_learning_rate=1e-3,
update_momentum=0.9,
verbose=1)
net.fit(X, y)
def classif_no_valid(self, NeuralNet, X, y):
from nolearn.lasagne import TrainSplit
l = InputLayer(shape=(None, X.shape[1]))
l = DenseLayer(l, num_units=len(np.unique(y)), nonlinearity=softmax)
net = NeuralNet(
l, update_learning_rate=0.01, train_split=TrainSplit(0))
return net.fit(X, y)
def net_fitted(self, NeuralNet, X_train, y_train):
nn = NeuralNet(
layers=[
('input', InputLayer),
('conv1', Conv2DLayer),
('conv2', Conv2DLayer),
('pool2', MaxPool2DLayer),
('output', DenseLayer),
],
input_shape=(None, 1, 28, 28),
output_num_units=10,
output_nonlinearity=softmax,
more_params=dict(
conv1_filter_size=(5, 5), conv1_num_filters=16,
conv2_filter_size=(3, 3), conv2_num_filters=16,
pool2_pool_size=(8, 8),
hidden1_num_units=16,
),
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
max_epochs=3,
)
return nn.fit(X_train, y_train)
input_shape=(None, 1, PIXELS, PIXELS),
conv1_num_filters=32, conv1_filter_size=(3, 3), pool1_ds=(2, 2),
conv2_num_filters=64, conv2_filter_size=(2, 2), pool2_ds=(2, 2),
hidden4_num_units=500,
output_num_units=2, output_nonlinearity=nonlinearities.softmax,
update_learning_rate=0.01,
update_momentum=0.9,
regression=False,
max_epochs=1000,
verbose=1,
)
class SimpleBatchIterator(BatchIterator):
def transform(self, Xb, yb):
Xb, yb = super(SimpleBatchIterator, self).transform(Xb, yb)
# The 'incomming' and outcomming shape is (batchsize, 1, 28, 28)
return Xb[:,:,::-1,:], yb #<--- Here we do the flipping
net1.batch_iterator_train = SimpleBatchIterator(batch_size=128)
net1.fit(X, y)
hidden3_nonlinearity=rectify, # Output Layer output_num_units=3, output_nonlinearity=softmax, # Optimization update=nesterov_momentum, update_learning_rate=0.001, update_momentum=0.3, max_epochs=30, # Others, regression=False, verbose=1, ) # # Train the NN print 'Training Neural Network' nnet.fit(X_train, y_train) print # Make predictions y_predictions = nnet.predict(X_test) print "f1 score:", f1_score(y_test, y_predictions, average='weighted') print classification_report(y_test, y_predictions) with open(r"../Pickled-Models/three-artist-neural-network.pickle", "wb") as output_file: cPickle.dump(nnet, output_file, protocol=cPickle.HIGHEST_PROTOCOL)
dropout2_p=0.5,
# output
output_nonlinearity=lasagne.nonlinearities.softmax,
output_num_units=10,
# optimization method params
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
max_epochs=10,
verbose=1,
)
# In[17]:
nn1 = net1.fit(X_train1, y_train)
# In[22]:
X_train_rescaling=X_train1/255
# In[23]:
net2 = NeuralNet(
layers=[('input', layers.InputLayer),
('conv2d1', layers.Conv2DLayer),
('maxpool1', layers.MaxPool2DLayer),
('conv2d2', layers.Conv2DLayer),
('maxpool2', layers.MaxPool2DLayer),
class RegressionUniformBlending(RegressionBase.RegressionBase):
def __init__(self, isTrain):
super(RegressionUniformBlending, self).__init__(isTrain)
# data preprocessing
#self.dataPreprocessing()
self.net1 = NeuralNet(
layers=[ # three layers: one hidden layer
('input', layers.InputLayer),
('hidden', layers.DenseLayer),
#('hidden2', layers.DenseLayer),
#('hidden3', layers.DenseLayer),
('output', layers.DenseLayer),
],
# layer parameters:
input_shape=(None, 13), # input dimension is 13
hidden_num_units=6, # number of units in hidden layer
#hidden2_num_units=8, # number of units in hidden layer
#hidden3_num_units=4, # number of units in hidden layer
output_nonlinearity=None, # output layer uses sigmoid function
output_num_units=1, # output dimension is 1
# obejctive function
objective_loss_function = lasagne.objectives.squared_error,
# optimization method:
update=lasagne.updates.nesterov_momentum,
update_learning_rate=0.002,
update_momentum=0.4,
# use 25% as validation
train_split=TrainSplit(eval_size=0.2),
regression=True, # flag to indicate we're dealing with regression problem
max_epochs=100, # we want to train this many epochs
verbose=0,
)
# Create linear regression object
self.linRegr = linear_model.LinearRegression()
# Create KNN regression object
self.knn = neighbors.KNeighborsRegressor(86, weights='distance')
# Create Decision Tree regression object
self.decisionTree = DecisionTreeRegressor(max_depth=7, max_features=None)
# Create AdaBoost regression object
decisionReg = DecisionTreeRegressor(max_depth=10)
rng = np.random.RandomState(1)
self.adaReg = AdaBoostRegressor(decisionReg,
n_estimators=400,
random_state=rng)
# Create linear regression object
self.model = RandomForestRegressor(max_features='sqrt', n_estimators=32, max_depth=39)
def dataPreprocessing(self):
# due to the observation, standization does not help the optimization.
# So do not use it!
#self.Standardization()
pass
def training(self):
# train each regression model
self.net1.fit(self.X_train, self.y_train)
self.linRegr.fit(self.X_train, self.y_train)
self.knn.fit(self.X_train, self.y_train)
self.decisionTree.fit(self.X_train, self.y_train)
self.adaReg.fit(self.X_train, self.y_train.reshape((self.y_train.shape[0], )))
def predict(self):
# predict the test data
y_pred1 = self.net1.predict(self.X_test)
y_pred1 = y_pred1.reshape((y_pred1.shape[0], 1))
y_pred2 = self.linRegr.predict(self.X_test)
y_pred2 = y_pred2.reshape((y_pred2.shape[0], 1))
y_pred3 = self.knn.predict(self.X_test)
y_pred3 = y_pred3.reshape((y_pred3.shape[0], 1))
y_pred4 = self.decisionTree.predict(self.X_test)
y_pred4 = y_pred4.reshape((y_pred4.shape[0], 1))
y_pred5 = self.adaReg.predict(self.X_test)
y_pred5 = y_pred5.reshape((y_pred5.shape[0], 1))
self.y_pred = (y_pred1+y_pred2+y_pred3+y_pred4+y_pred5)/5
# print MSE
mse = mean_squared_error(self.y_pred, self.y_test)
print "MSE: {}".format(mse)
('dropout2', layers.DropoutLayer),
('output', layers.DenseLayer),
],
input_shape=(None, 1, 28, 28),
conv2d1_num_filters=32,
conv2d1_filter_size=(5, 5),
conv2d1_nonlinearity=lasagne.nonlinearities.rectify,
conv2d1_W=lasagne.init.GlorotUniform(),
maxpool1_pool_size=(2, 2),
conv2d2_num_filters=32,
conv2d2_filter_size=(5, 5),
conv2d2_nonlinearity=lasagne.nonlinearities.rectify,
maxpool2_pool_size=(2, 2),
dropout1_p=0.5,
dense_num_units=256,
dense_nonlinearity=lasagne.nonlinearities.rectify,
dropout2_p=0.5,
output_nonlinearity=lasagne.nonlinearities.softmax,
output_num_units=10,
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
max_epochs=10,
verbose=1,
)
nn = CNN.fit(X_train, y_train)
prediction = CNN.predict(X_test)
visualize.plot_conv_weights(CNN.layers_['conv2d1'])
def classif(self, NeuralNet, X, y):
l = InputLayer(shape=(None, X.shape[1]))
l = DenseLayer(l, num_units=len(np.unique(y)), nonlinearity=softmax)
net = NeuralNet(l, update_learning_rate=0.01)
return net.fit(X, y)
conv2d1_W=lasagne.init.GlorotUniform(),
# layer maxpool1
maxpool1_pool_size=(2, 2),
# layer conv2d2
conv2d2_num_filters=32,
conv2d2_filter_size=(5, 5),
conv2d2_nonlinearity=lasagne.nonlinearities.rectify,
# layer maxpool2
maxpool2_pool_size=(2, 2),
# dropout1
dropout1_p=0.5,
# dense
dense_num_units=256,
dense_nonlinearity=lasagne.nonlinearities.rectify,
# dropout2
dropout2_p=0.5,
# output
output_nonlinearity=lasagne.nonlinearities.softmax,
output_num_units=10,
# optimization method params
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
max_epochs=10,
verbose=1,
)
# Train the network
nn = net1.fit(X_train, y_train)
visualize.plot_conv_weights(net1.layers_['conv2d1'])
def regr(self, NeuralNet, X, y):
l = InputLayer(shape=(None, X.shape[1]))
l = DenseLayer(l, num_units=y.shape[1], nonlinearity=None)
net = NeuralNet(l, regression=True, update_learning_rate=0.01)
return net.fit(X, y)
y_tensor_type=T.tensor4,
#optimization parameters:
update=nesterov_momentum,
update_learning_rate=0.05,
update_momentum=0.9,
regression=True,
max_epochs=50,
verbose=1,
batch_iterator_train=BatchIterator(batch_size=25),
on_epoch_finished=[
EarlyStopping(patience=20),
],
train_split=TrainSplit(eval_size=0.5))
net2.fit(train_x, train_y)
plot_loss(net2)
plt.savefig("/home/sandeep/Desktop/Major/Results/plotloss.png")
plot_conv_weights(net2.layers_[1], figsize=(4, 4))
plt.savefig("/home/sandeep/Desktop/Major/Results/convweights.png")
#layer_info = PrintLayerInfo()
#layer_info(net2)
import cPickle as pickle
with open('/home/sandeep/Desktop/Major/Results/net2.pickle', 'wb') as f:
pickle.dump(net2, f, -1)
y_pred2 = net2.predict(test_x)
print "The accuracy of this network is: %0.2f" % (abs(y_pred2 - test_y)).mean()
# update=adagrad,
# update_learning_rate=.1,
update=nesterov_momentum,
update_learning_rate=theano.shared(np.float32(0.01)),
update_momentum=theano.shared(np.float32(0.9)),
#
on_epoch_finished=[
AdjustVariable('update_learning_rate', start=0.01, stop=0.00001),
AdjustVariable('update_momentum', start=0.9, stop=0.999),
],
regression=
False, # flag to indicate we're dealing with regression problem
max_epochs=500, # we want to train this many epochs
verbose=1)
clf.fit(x_train, y_train)
# if 1:
# y_pred = clf.predict_proba(x_test)
#
# filename = 'testdata_aug_d'
# savefile = open(filename+'.pkl', 'wb')
# cPickle.dump((x_test, y_pred, name1),savefile,-1)
# savefile.close()
if 1:
y_pred = clf.predict(x_test)
print "Accuracy:", zero_one_loss(y_test, y_pred)
print "Classification report:"
print classification_report(y_test, y_pred)
print 'Confusion matrix:'
print confusion_matrix(y_test, y_pred)
conv2d1_num_filters=20, #20 filters, same for each channel?
conv2d1_filter_size=(5, 5),
conv2d1_stride=(1, 1),
conv2d1_pad=(2, 2),
conv2d1_nonlinearity=lasagne.nonlinearities.rectify, #standard reLu
maxpool1_pool_size=(2, 2),
conv2d2_num_filters=20,
conv2d2_filter_size=(5, 5),
conv2d2_stride=(1, 1),
conv2d2_pad=(2, 2),
conv2d2_nonlinearity=lasagne.nonlinearities.rectify,
maxpool2_pool_size=(2, 2),
dense_num_units=1000,
dense_nonlinearity=lasagne.nonlinearities.rectify,
output_nonlinearity=lasagne.nonlinearities.softmax,
output_num_units=10,
update=nesterov_momentum,
update_momentum=0.9,
update_learning_rate=0.0001,
max_epochs=100,
verbose=True)
x_train, y_train, x_test, y_test = load_data(
os.path.expanduser('~/deep-learning/data/cifar-10-batches-py'))
network = net.fit(x_train, y_train)
predictions = network.predict(x_test)
print classification_report(y_test, predictions)
print accuracy_score(y_test, predictions)
def main():
data = load_av_letters('data/allData_mouthROIs.mat')
# create the necessary variable mappings
data_matrix = data['dataMatrix']
data_matrix_len = data_matrix.shape[0]
targets_vec = data['targetsVec']
vid_len_vec = data['videoLengthVec']
iter_vec = data['iterVec']
indexes = create_split_index(data_matrix_len, vid_len_vec, iter_vec)
# split the data
train_data = data_matrix[indexes == True]
train_targets = targets_vec[indexes == True]
test_data = data_matrix[indexes == False]
test_targets = targets_vec[indexes == False]
idx = [i for i, elem in enumerate(test_targets) if elem == 20]
print(train_data.shape)
print(test_data.shape)
print(sum([train_data.shape[0], test_data.shape[0]]))
# resize the input data to 40 x 30
train_data_resized = resize_images(train_data).astype(np.float32)
# normalize the inputs [0 - 1]
train_data_resized = normalize_input(train_data_resized, centralize=True)
test_data_resized = resize_images(test_data).astype(np.float32)
test_data_resized = normalize_input(test_data_resized, centralize=True)
dic = {}
dic['trainDataResized'] = train_data_resized
dic['testDataResized'] = test_data_resized
"""second experiment: overcomplete sigmoid encoder/decoder, squared loss"""
encode_size = 2500
sigma = 0.5
# to get tied weights in the encoder/decoder, create this shared weightMatrix
# 1200 x 2000
w1, layer1 = build_encoder_layers(1200, 2500, sigma)
ae1 = NeuralNet(layers=layer1,
max_epochs=50,
objective_loss_function=squared_error,
update=adadelta,
regression=True,
verbose=1)
load = True
save = False
if load:
print('[LOAD] layer 1...')
ae1.load_params_from('layer1.dat')
else:
print('[TRAIN] layer 1...')
ae1.fit(train_data_resized, train_data_resized)
# save params
if save:
print('[SAVE] layer 1...')
ae1.save_params_to('layer1.dat')
train_encoded1 = ae1.get_output('encoder',
train_data_resized) # 12293 x 2000
w2, layer2 = build_encoder_layers(2500, 1250)
ae2 = NeuralNet(layers=layer2,
max_epochs=50,
objective_loss_function=squared_error,
update=adadelta,
regression=True,
verbose=1)
load2 = True
if load2:
print('[LOAD] layer 2...')
ae2.load_params_from('layer2.dat')
else:
print('[TRAIN] layer 2...')
ae2.fit(train_encoded1, train_encoded1)
save2 = False
if save2:
print('[SAVE] layer 2...')
ae2.save_params_to('layer2.dat')
train_encoded2 = ae2.get_output('encoder', train_encoded1) # 12293 x 1250
w3, layer3 = build_encoder_layers(1250, 600)
ae3 = NeuralNet(layers=layer3,
max_epochs=100,
objective_loss_function=squared_error,
update=adadelta,
regression=True,
verbose=1)
load3 = True
if load3:
print('[LOAD] layer 3...')
ae3.load_params_from('layer3.dat')
else:
ae3.fit(train_encoded2, train_encoded2)
save3 = False
if save3:
print('[SAVE] layer 3...')
ae3.save_params_to('layer3.dat')
train_encoded3 = ae3.get_output('encoder', train_encoded2) # 12293 x 1250
w4, layer4 = build_bottleneck_layer(600, 100)
ae4 = NeuralNet(layers=layer4,
max_epochs=100,
objective_loss_function=squared_error,
update=adadelta,
regression=True,
verbose=1)
load4 = False
if load4:
print('[LOAD] layer 4...')
ae4.load_params_from('layer4.dat')
else:
print('[TRAIN] layer 4...')
ae4.fit(train_encoded3, train_encoded3)
save4 = True
if save4:
print('[SAVE] layer 4...')
ae4.save_params_to('layer4.dat')
test_enc1 = ae1.get_output('encoder', test_data_resized)
test_enc2 = ae2.get_output('encoder', test_enc1)
test_enc3 = ae3.get_output('encoder', test_enc2)
test_enc4 = ae4.get_output('encoder', test_enc3)
decoder4 = create_decoder(100, 600, w4.T)
decoder4.initialize()
decoder3 = create_decoder(600, 1250, w3.T)
decoder3.initialize()
decoder2 = create_decoder(1250, 2500, w2.T)
decoder2.initialize() # initialize the net
decoder1 = create_decoder(2500, 1200, w1.T)
decoder1.initialize()
test_dec3 = decoder4.predict(test_enc4)
test_dec2 = decoder3.predict(test_dec3)
test_dec1 = decoder2.predict(test_dec2)
X_pred = decoder1.predict(test_dec1)
# plot_loss(ae3)
# plot_loss(ae2)
# plot_loss(ae1)
tile_raster_images(X_pred[4625:4650, :], (30, 40), (5, 5),
tile_spacing=(1, 1))
plt.title('reconstructed')
tile_raster_images(test_data_resized[4625:4650, :], (30, 40), (5, 5),
tile_spacing=(1, 1))
plt.title('original')
plt.show()
"""
update_learning_rate=0.05,
update_momentum=0.5,
max_epochs=25,
#update = adagrad,
#update_learning_rate = .07,
#max_epochs = 50,
# Others
regression=False,
verbose=1,
)
# In[221]:
nn.fit(train_x, train_y)
predict_y = nn.predict(test_x)
print "Accuracy Score: " +str(accuracy_score(test_y, predict_y))
print "Precision Score: " + str(precision_score(test_y, predict_y))
print "Recall Score: " + str(recall_score(test_y, predict_y))
print "F1 Score: " + str(f1_score(test_y, predict_y))
# In[ ]:
max_epochs=200)
net2 = NeuralNet(layers=[('input', layers.InputLayer),
('conv1', layers.Conv2DLayer),
('pool1', layers.MaxPool2DLayer),
('conv2', layers.Conv2DLayer),
('pool2', layers.MaxPool2DLayer),
('conv3', layers.Conv2DLayer),
('pool3', layers.MaxPool2DLayer),
('hidden4', layers.DenseLayer),
('hidden5', layers.DenseLayer),
('output', layers.DenseLayer),],
input_shape=(None, 1, 1, num_features),
conv1_num_filters=32, conv1_filter_size=(1, 3), pool1_ds=(1, 2),
conv2_num_filters=64, conv2_filter_size=(1, 2), pool2_ds=(1, 2),
conv3_num_filters=128, conv3_filter_size=(1, 2), pool3_ds=(1, 2),
hidden4_num_units=500, hidden5_num_units=500,
output_num_units=num_classes, output_nonlinearity=softmax,
update_learning_rate=0.01,
update_momentum=0.9,
max_epochs=100,
verbose=1,
)
##X = X.reshape(-1, 1, 1, num_features)
##X_test = X_test.reshape(-1, 1, 1, num_features)
net0.fit(X, y)
make_submission(net0, X_test, ids, encoder)
lr = 1e-4
momentum = 0.9
net = NeuralNet(
layers=[
('i', layers.InputLayer),
('c1', Conv2DLayer),
('p1', MaxPool2DLayer),
('o', layers.DenseLayer),
],
i_shape=(None, ) + ds.X.shape[1:],
c1_num_filters=3,
c1_filter_size=(3, 3),
p1_pool_size=(2, 2),
o_num_units=len(set(ds.y)),
o_nonlinearity=nonlinearities.softmax,
update_learning_rate=theano.shared(np.float32(lr)),
update_momentum=theano.shared(np.float32(momentum)),
regression=False,
on_epoch_finished=[
AdjustVariable('update_learning_rate', start=lr, stop=0.0001),
AdjustVariable('update_momentum', start=momentum, stop=0.999),
EarlyStopping(patience=200),
],
max_epochs=3000,
verbose=1,
)
net.fit(ds.X, ds.y)
save_net(net)
regression=False,
#loss=lasagne.objectives.multinomial_nll,
use_label_encoder=True,
batch_iterator_train=FlipBatchIterator(batch_size=256),
on_epoch_finished=[
AdjustVariable('update_learning_rate', start=0.01, stop=0.0001),
AdjustVariable('update_momentum', start=0.9, stop=0.999),
EarlyStopping(patience=50),
],
max_epochs=500,
verbose=2,
)
if TRAIN:
net2.fit(X, y)
with open(NETWORK, 'wb') as f:
pickle.dump(net2, f, -1)
def predict():
with open(NETWORK, 'rb') as f:
model = pickle.load(f)
testdata = load_test()
predictions = np.zeros((testdata.shape[0], 121))
for i in range(0, testdata.shape[0], 4075):
preds = model.predict_proba(testdata[i:i+4075,:].astype(np.float32).reshape(-1, 1, IMG_SIZE, IMG_SIZE))
predictions[i:i+4075,:] = preds
print "Done: ", i+4075
return model, predictions
dense3_nonlinearity=nonlinearities.sigmoid,
#dense4_nonlinearity = nonlinearities.sigmoid,
output_nonlinearity=softmax,
update=nesterov_momentum,
update_learning_rate=0.03,
eval_size=0.2,
verbose=1,
max_epochs=140,
regression=False)
clf = NeuralNet(**params)
print "PARAMS : "
print params
clf.fit(X, Y)
file_valid = pd.read_hdf("test.h5", "test")
file_valid = np.array(file_valid)
data = np.array([row[0:] for row in file_valid])
data = skpre.StandardScaler().fit_transform(data)
number = range(45324, 53461)
predict = clf.predict(data)
with open('result.csv', 'wb') as csvfile:
fieldnames = ['Id', 'y']
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
# check if a trained model already exists
cnn.load_params_from('./data_cache/combined_dataset/cnn_' + CNNCode + '-' +
TrainCode + '.pkl')
else:
cnn_train = time.time()
# training a new model
for epoch in range(Epochs):
# for every epoch
for batch in patches_extract_all(Train):
# for every batch
inputs, targets = batch
# data augmentation
inputs, targets = data_aug(inputs, targets)
# run cnn.fit for 1 iteration
#cnn_fit = time.time();
cnn.fit(inputs, targets.reshape((-1, 1 * 32 * 32)))
# print 'fitting cnn took: ', time.time()-cnn_fit, 'sec.';
# for every 10 epoch, print testing accuracy
'''if epoch % 10 == 0:
for batch in patches_extract_all(Test):
inputs, targets = batch;
predicts = cnn.predict(inputs);
T = targets.reshape((-1, 1 * 32 * 32)).flatten().astype(np.int32);
P = (predicts.flatten() > 0.5).astype(np.int32);
print("======================= {:.4f} =======================".format(accuracy_score(T, P)));'''
print('training cnn took: ',
time.time() - cnn_train, 'sec.')
# save the trained model
cnn.save_params_to('./data_cache/combined_dataset/cnn_' + CNNCode + '-' +
TrainCode + '.pkl')
cons = ({'type':'eq','fun':lambda w: 1-sum(w)})
bounds = [(0,1)]*6
weights = []
for train_index, test_index in re:
aucList = []
predictions = []
gbm = xgb.train(params,xgb.DMatrix(result.iloc[train_index,:],y.iloc[train_index]),num_trees)
rf.fit(X_train[train_index,:],y[train_index])
ada.fit(X_train[train_index,:],y[train_index])
svr.fit(X_train[train_index,:],y[train_index])
knn.fit(X_train[train_index,:],y[train_index])
y_train = encoder.fit_transform(y[train_index]).astype(np.int32)
net0.fit(X_train[train_index,:],y_train)
predictions.append(svr.predict_proba(X_train[test_index])[:,1])
predictions.append(gbm.predict(xgb.DMatrix(result.iloc[test_index])))
predictions.append(ada.predict_proba(X_train[test_index])[:,1])
predictions.append(rf.predict_proba(X_train[test_index])[:,1])
predictions.append(knn.predict_proba(X_train[test_index])[:,1])
predictions.append(net0.predict_proba(X_train[test_index])[:,1])
netAccuracy.append(roc_auc_score(y.iloc[test_index],net0.predict_proba(X_train[test_index])[:,1]))
starting_values = [1./6]*len(predictions)
# train_eval_probs = 0.30*svr.predict_proba(X_train[test_index])[:,1] + 0.15*gbm.predict(xgb.DMatrix(result.iloc[test_index])) \
# + 0.15*ada.predict_proba(X_train[test_index])[:,1] + 0.35*rf.predict_proba(X_train[test_index])[:,1] \
# + 0.05*knn.predict_proba(X_train[test_index])[:,1]
res = minimize(log_loss_func, starting_values, method='SLSQP', bounds=bounds, constraints=cons)
print('Ensamble Score: {best_score}'.format(best_score=res['fun']))
print('Best Weights: {weights}'.format(weights=res['x']))
def run_cross_validation(nfolds=10):
unique_drivers = trainingLabels['subject'].unique()
#driver_id = {}
#for d_id in unique_drivers:
#driver_id[d_id] = trainingLabels.loc[trainingLabels['subject'] == d_id]['img']
driver_id = trainingLabels['subject']
#print('Unique drivers ' + str(unique_drivers) + '\n' + str(driver_id))
kf = cross_validation.KFold(len(unique_drivers), n_folds=nfolds, shuffle=True)
num_fold = 1
yfull_train = dict()
yfull_test = []
for train_drivers, test_drivers in kf:
unique_list_train = [unique_drivers[i] for i in train_drivers]
#print('Unique drivers train ' + str(unique_list_train))
X_train, Y_train, train_index = copy_selected_drivers(xTrain, y, driver_id, unique_list_train)
unique_list_valid = [unique_drivers[i] for i in test_drivers]
#print('Unique drivers validation ' + str(unique_list_valid))
X_valid, Y_valid, test_index = copy_selected_drivers(xTrain, y, driver_id, unique_list_valid)
outputLayer = None
if source == '1':
outputLayer = getNet1()
#outputLayer = getNet2()
#outputLayer = getNet8()
numberEpochs = 30
elif source == '2':
outputLayer = getNet3()
numberEpochs = 30
elif source == '3':
outputLayer = getNet4()
#outputLayer = getNet5()
#outputLayer = getNet6()
#outputLayer = getNet7()
numberEpochs = 10
elif source == '4':
outputLayer = getNet9()
#outputLayer = getNet10()
numberEpochs = 30
net = NeuralNet(
layers = outputLayer,
update=updates.nesterov_momentum,
#update=updates.adam,
#update=updates.rmsprop,
#update=updates.adadelta,
update_learning_rate = 0.001,
#update_beta1 = 0.9,
#update_beta2 = 0.999,
#update_epsilon = 1e-8,
update_momentum = 0.9,
#update_rho=0.95,
#update_epsilon=1e-06,
objective_loss_function = objectives.categorical_crossentropy,
#objective=objectives.categorical_crossentropy,
batch_iterator_train = BatchIterator(batch_size = batchSize),
batch_iterator_test = BatchIterator(batch_size = batchSize),
#custom_scores = [objectives.categorical_accuracy],
use_label_encoder = True,
#use_label_encoder = False,
regression = False,
max_epochs = numberEpochs,
verbose = 1
)
net.fit(X_train, Y_train)
predictValidY = net.predict_proba(X_valid)
score = metrics.log_loss(Y_valid, predictValidY)
print('Fold ' + str(num_fold) + ' score ' + str(score))
for i in range(len(test_index)):
yfull_train[test_index[i]] = predictValidY[i]
test_prediction = net.predict_proba(xTest)
yfull_test.append(test_prediction)
if gcEnabled:
gc.collect()
num_fold += 1
score = metrics.log_loss(y, dict_to_list(yfull_train))
print('Final log loss ' + str(score))
test_res = merge_several_folds_mean(yfull_test, nfolds)
predictions = pd.DataFrame(test_res, index=files)
predictions.index.name = 'img'
predictions.columns = ['c0', 'c1', 'c2', 'c3', 'c4', 'c5', 'c6', 'c7', 'c8', 'c9']
predictions.to_csv(predictionsFile)
dropout2_p = 0.5,
# output layer descriptors
output_nonlinearity = lasagne.nonlinearities.softmax,
output_num_units=10,
#optimization parameters
update=nesterov_momentum,
update_learning_rate=0.01,
max_epochs=max_epochs
)
# In[10]:
print('training')
nn = net.fit(X_train, Y_train)
print('training complete')
# In[11]:
# In[12]:
X_test = np.load('./data/test_inputs.npy')
# In[13]:
X_test = X_test.reshape(-1, 1, PIXELS, PIXELS)
('dropout4', DropoutLayer),
('output', DenseLayer),
],
input_shape=(None, 1, 28, 28),
conv1_num_filters=32,
conv1_filter_size=(3, 3),
pool1_pool_size=(2, 2),
dropout1_p=0.2,
conv2_num_filters=64,
conv2_filter_size=(3, 3),
pool2_pool_size=(2, 2),
dropout2_p=0.2,
hidden3_num_units=1024,
dropout3_p=0.5,
hidden4_num_units=1024,
dropout4_p=0.5,
output_num_units=num_classes,
output_nonlinearity=softmax,
update=adagrad,
update_learning_rate=theano.shared(float32(0.03)),
eval_size=0.01,
on_epoch_finished=[
AdjustVariable('update_learning_rate', start=0.01, stop=0.0001),
],
max_epochs=30,
verbose=1,
)
net.fit(X, y)
nombre = "LasagneCNN.csv"
make_submission(net, X_test, encoder, nombre)
hidden_num_units=100, # number of units in hidden layer
output_nonlinearity=None, # output layer uses identity function
output_num_units=30, # 30 target values
# optimization method:
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
regression=True, # flag to indicate we're dealing with regression problem
max_epochs=400, # we want to train this many epochs
verbose=1,
)
X, y = load()
net1.fit(X, y)
def plot_sample(x, y, axis): # plotting the images with predicted keypoints
img = x.reshape(96, 96)
axis.imshow(img, cmap='gray')
axis.scatter(y[0::2] * 48 + 48, y[1::2] * 48 + 48, marker='x', s=10)
X, _ = load(test=True)
y_pred = net1.predict(X)
fig = pyplot.figure(figsize=(6, 6))
fig.subplots_adjust(
left=0, right=1, bottom=0, top=1, hspace=0.05, wspace=0.05)
for i in range(16):
ax = fig.add_subplot(4, 4, i + 1, xticks=[], yticks=[])
def NNet(X,y,test):
#rows = int(X.shape[0])
cols = int(X.shape[1])
net = NeuralNet(
layers = [
('input',layers.InputLayer),
('hidden1',layers.DenseLayer),
('dropout1',layers.DropoutLayer),
('hidden2',layers.DenseLayer),
('dropout2',layers.DropoutLayer),
('hidden3',layers.DenseLayer),
('dropout3',layers.DropoutLayer),
('hidden4',layers.DenseLayer),
#('dropout4',layers.DropoutLayer),
('output',layers.DenseLayer),
],
input_shape = (None,cols),
hidden1_num_units = 800,
dropout1_p = 0.4,
hidden2_num_units = 500,
dropout2_p = 0.3,
hidden3_num_units = 300,
dropout3_p = 0.3,
hidden4_num_units = 200,
#dropout4_p = 0.2,
output_num_units = len(np.unique(y)),
output_nonlinearity = lasagne.nonlinearities.softmax,
update=nesterov_momentum,
update_learning_rate=0.0001,
update_momentum=0.9,
max_epochs = 300,
verbose = 1,
)
# net.load_params_from('w3')
# details = net.get_all_params()
# oldw = net.get_all_params_values()
'''
skf = cross_validation.StratifiedKFold(y,n_folds = 7)
blend_train = np.zeros(X.shape[0])
prediction = []
blend_test_j = np.zeros((test.shape[0], len(skf)))
for i,(train_index,cv_index) in enumerate(skf):
print "Fold:",i
X_train = X[train_index,]
y_train = y[train_index]
X_cv = X[cv_index,]
#y_cv = y[cv_index]
net.fit(X_train,y_train)
blend_train[cv_index] = net.predict_proba(X_cv)[:,1]
blend_test_j[:,i] = net.predict_proba(test)[:,1]
prediction = blend_test_j.mean(1)
'''
net.fit(X,y)
prediction = net.predict_proba(test)
return prediction
dense_nonlinearity=lasagne.nonlinearities.rectify,
# dropout2
dropout2_p=0.5,
# output
output_nonlinearity=lasagne.nonlinearities.softmax,
output_num_units=26,
# optimization method params
update=nesterov_momentum,
update_learning_rate=0.02,
update_momentum=0.9,
max_epochs=50,
verbose=1,
)
# Train the network
nn = net1.fit(X_train_s, y_train_s)
#nn = net1.fit(X_train, y_train)
# Test CNN
preds = net1.predict(X_test)
# Confusion matrix
plot_confusion_matrix(y_test, preds)
nn = net1.fit(X_train, y_train)
# Test CNN
preds = net1.predict(X_test)
# Confusion matrix
plot_confusion_matrix(y_test, preds)
conv3_num_filters=128, conv3_filter_size=(2, 2), pool3_pool_size=(2, 2),
hidden4_num_units=500,
hidden5_num_units=500,
output_num_units=30, output_nonlinearity=None,
update_learning_rate=theano.shared(float32(0.03)),
update_momentum=theano.shared(float32(0.9)),
regression=True,
batch_iterator_train=FlipBatchIterator(batch_size=128),
on_epoch_finished=[
AdjustVariable('update_learning_rate', start=0.03, stop=0.0001),
AdjustVariable('update_momentum', start=0.9, stop=0.999),
],
max_epochs=3000,
verbose=1,
)
#####################
#####################
#####################
X, y = load2d() # load 2-d data
net5.fit(X, y)
import cPickle as pickle
with open('net5.pickle', 'wb') as f:
pickle.dump(net5, f, -1)
def smart_find(X, y, X_valid, y_valid):
loss = []
kf = KFold(n_splits=5, shuffle=True)
conf_set = set()
step = (64 + 10) / 4
max_neuron_units = step * 8
for i in range(1, max_neuron_units, step):
for j in range(0, max_neuron_units, step):
for k in range(0, max_neuron_units, step):
struct_net = (i)
l = InputLayer(shape=(None, X.shape[1]))
# ------- HIDDEN -----------
l = DenseLayer(l, num_units=i, nonlinearity=softmax)
if j > 0:
if i + step < j:
continue
l = DenseLayer(l, num_units=j, nonlinearity=softmax)
struct_net = (i, j)
if k > 0:
if i + step < k or j + step < k:
continue
struct_net = (i, j, k)
l = DenseLayer(l, num_units=k, nonlinearity=softmax)
# ------- HIDDEN -----------
l = DenseLayer(l,
num_units=len(np.unique(y)),
nonlinearity=softmax)
net = NeuralNet(l,
update=adam,
update_learning_rate=0.01,
max_epochs=250)
if struct_net in conf_set:
continue
print('=' * 40)
print(struct_net)
print('=' * 40)
conf_set.add(struct_net)
k_loss = []
y_data = np.array([y]).transpose()
data = np.concatenate((X, y_data), axis=1)
for train_index, test_index in kf.split(data):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
net.fit(X_train, y_train)
y_pred = net.predict(X_test)
loss_error = net.score(X_test, y_test)
# loss_error = mean_squared_error(y_test, y_pred)
k_loss.append(loss_error)
print(loss_error)
loss_net = (i, j, k, np.array(k_loss).mean())
print(loss_net)
loss.append(loss_net)
print('=' * 40)
# for i in range(1, max_hidden_layers):
# for j in range((64 + 10) // 2 // i, max_neuron_units // i, 10 // i):
# print('=' * 40)
# print('%s hidden layers' % i)
# print('%s neurons' % j)
# print('=' * 40)
# l = InputLayer(shape=(None, X.shape[1]))
# for k in range(i):
# l = DenseLayer(l, num_units=j, nonlinearity=softmax)
# l = DenseLayer(l, num_units=len(np.unique(y)), nonlinearity=softmax)
# net = NeuralNet(l, update=adam, update_learning_rate=0.01, max_epochs=500)
#
# k_loss = []
# y_data = np.array([y]).transpose()
# data = np.concatenate((X, y_data), axis=1)
# for train_index, test_index in kf.split(data):
# X_train, X_test = X[train_index], X[test_index]
# y_train, y_test = y[train_index], y[test_index]
#
# net.fit(X_train, y_train)
# y_pred = net.predict(X_test)
# loss_error = mean_squared_error(y_test, y_pred)
# k_loss.append(loss_error)
# print(loss_error)
#
# loss_net = (i, j, np.array(k_loss).mean())
# print(loss_net)
# loss.append(loss_net)
# print('=' * 40)
print(min(loss, key=lambda x: x[3]))
print(max(loss, key=lambda x: x[3]))
print(loss)
class ClassificationLinearBlending(ClassficationBase.ClassificationBase):
def __init__(self, isTrain, isOutlierRemoval=0):
super(ClassificationLinearBlending, self).__init__(isTrain, isOutlierRemoval)
# data preprocessing
self.dataPreprocessing()
# create logistic regression object
self.logreg = linear_model.LogisticRegression(tol=1e-6, penalty='l1', C=0.0010985411419875584)
# create adaboost object
self.dt_stump = DecisionTreeClassifier(max_depth=10)
self.ada = AdaBoostClassifier(
base_estimator=self.dt_stump,
learning_rate=1,
n_estimators=5,
algorithm="SAMME.R")
# create knn object
self.knn = neighbors.KNeighborsClassifier(6, weights='uniform')
# create decision tree object
self.decisiontree = DecisionTreeClassifier(max_depth=50)
# create neural network object
self.net1 = NeuralNet(
layers=[ # three layers: one hidden layer
('input', layers.InputLayer),
('hidden', layers.DenseLayer),
#('hidden2', layers.DenseLayer),
('output', layers.DenseLayer),
],
# layer parameters:
input_shape=(None, 12), # inut dimension is 12
hidden_num_units=6, # number of units in hidden layer
#hidden2_num_units=3, # number of units in hidden layer
output_nonlinearity=lasagne.nonlinearities.sigmoid, # output layer uses sigmoid function
output_num_units=1, # output dimension is 1
# optimization method:
update=nesterov_momentum,
update_learning_rate=0.002,
update_momentum=0.9,
regression=True, # flag to indicate we're dealing with regression problem
max_epochs=25, # we want to train this many epochs
verbose=0,
)
def dataPreprocessing(self):
# deal with unbalanced data
self.dealingUnbalancedData()
# Standardization
#self.Standardization()
def training(self):
# train the models
self.logreg.fit(self.X_train, self.y_train.ravel())
self.ada.fit(self.X_train, self.y_train.reshape((self.y_train.shape[0], )))
self.knn.fit(self.X_train, self.y_train.ravel())
self.decisiontree.fit(self.X_train, self.y_train)
self.net1.fit(self.X_train, self.y_train)
def predict(self):
# predict the test data
y_pred1 = self.logreg.predict(self.X_test)
y_pred1 = y_pred1.reshape((y_pred1.shape[0], 1))
y_pred2 = self.ada.predict(self.X_test)
y_pred2 = y_pred2.reshape((y_pred2.shape[0], 1))
y_pred3 = self.knn.predict(self.X_test)
y_pred3 = y_pred3.reshape((y_pred3.shape[0], 1))
y_pred4 = self.decisiontree.predict(self.X_test)
y_pred4 = y_pred4.reshape((y_pred4.shape[0], 1))
# predict neural network
# predict the test data
y_pred_train = self.net1.predict(self.X_train)
y_pred5 = self.net1.predict(self.X_test)
# keep all the predictions
y_preds = []
y_preds.append(y_pred1)
y_preds.append(y_pred2)
y_preds.append(y_pred3)
y_preds.append(y_pred4)
y_preds.append(y_pred5)
# 1 for buy, 0 for wait
median = np.median(y_pred_train)
mean = np.mean(y_pred_train)
y_pred5[y_pred5>=median] = 1 # change this threshold
y_pred5[y_pred5<median] = 0
y_pred5 = y_pred5.reshape((y_pred5.shape[0], 1))
# get the error rate
self.y_pred = (y_pred2+y_pred3+y_pred4)/3
self.y_pred[self.y_pred >= 0.5] = 1
self.y_pred[self.y_pred < 0.5] = 0
e1 = 1 - np.sum(self.y_test == y_pred1) * 1.0 / y_pred1.shape[0]
e2 = 1 - np.sum(self.y_test == y_pred2) * 1.0 / y_pred2.shape[0]
e3 = 1 - np.sum(self.y_test == y_pred3) * 1.0 / y_pred3.shape[0]
e4 = 1 - np.sum(self.y_test == y_pred4) * 1.0 / y_pred4.shape[0]
e5 = 1 - np.sum(self.y_test == y_pred5) * 1.0 / y_pred5.shape[0]
e = 1 - np.sum(self.y_test == self.y_pred) * 1.0 / self.y_pred.shape[0]
print "e1 = {}".format(e1)
print "e2 = {}".format(e2)
print "e3 = {}".format(e3)
print "e4 = {}".format(e4)
print "e5 = {}".format(e5)
print "Uniform error = {}".format(e)
# keep all the error rates
errors = []
errors.append(e1)
errors.append(e2)
errors.append(e3)
errors.append(e4)
errors.append(e5)
# totalAlpha = 0
# self.y_pred = np.zeros(shape=self.y_pred.shape)
# for i in range(5):
# if errors[i] <= 0.5:
# alpha = math.log(math.sqrt((1-errors[i])/errors[i]))
# totalAlpha += alpha
# self.y_pred = self.y_pred + alpha*y_preds[i]
#
# # predict the blending output
# self.y_pred = self.y_pred / totalAlpha
# self.y_pred[self.y_pred >= 0.5] = 1
# self.y_pred[self.y_pred < 0.5] = 0
alpha2 = math.log(math.sqrt((1-e2)/e2))
alpha3 = math.log(math.sqrt((1-e3)/e3))
alpha4 = math.log(math.sqrt((1-e4)/e4))
self.y_pred = (alpha2*y_pred2 + alpha3*y_pred3 + alpha4*y_pred4) * 1.0 /(alpha2+alpha3+alpha4)
def train():
weather = load_weather()
training = load_training()
X = assemble_X(training, weather)
mean, std = normalize(X)
y = assemble_y(training)
col_labels = [
"date.year",
"date.month",
"date.day",
"block",
"lat",
"long",
]
for obs in [
"Tmax", "Tmin", "Tavg", "DewPoint", "WetBulb", "PrecipTotal",
"Depart", "Sunrise", "Sunset", "Heat", "Cool", "ResultSpeed",
"ResultDir"
]:
for day in ["1", "2", "3", "5", "8", "12"]:
col_labels.append(obs + "_" + day)
for i_spec in range(6):
col_labels.append("species_" + str(i_spec))
X_file = csv.writer(open("X.csv", "w"))
X_file.writerow(col_labels)
for row in X:
X_file.writerow(row)
y_file = csv.writer(open("y.csv", "w"))
y_file.writerow(["y"])
for row in y:
y_file.writerow(row)
input_size = len(X[0])
learning_rate = theano.shared(np.float32(0.1))
net = NeuralNet(
layers=[
('input', InputLayer),
('hidden1', DenseLayer),
('dropout1', DropoutLayer),
('hidden2', DenseLayer),
('dropout2', DropoutLayer),
('output', DenseLayer),
],
# layer parameters:
input_shape=(None, input_size),
hidden1_num_units=600,
dropout1_p=0.5,
hidden2_num_units=400,
dropout2_p=0.5,
output_nonlinearity=sigmoid,
output_num_units=1,
# optimization method:
update=nesterov_momentum,
update_learning_rate=learning_rate,
update_momentum=0.9,
# Decay the learning rate
on_epoch_finished=[
AdjustVariable(learning_rate, target=0, half_life=20),
],
# This is silly, but we don't want a stratified K-Fold here
# To compensate we need to pass in the y_tensor_type and the loss.
regression=True,
y_tensor_type=T.imatrix,
objective_loss_function=binary_crossentropy,
max_epochs=500,
eval_size=0.1,
verbose=1,
)
X, y = shuffle(X, y, random_state=123)
net.fit(X, y)
_, X_valid, _, y_valid = net.train_test_split(X, y, net.eval_size)
probas = net.predict_proba(X_valid)[:, 0]
print("ROC score", metrics.roc_auc_score(y_valid, probas))
return net, mean, std
update_momentum=theano.shared(np.float32(0.9)),
# Decay the learning rate
on_epoch_finished=[
AdjustVariable('update_learning_rate', start=0.001, stop=0.0001),
AdjustVariable('update_momentum', start=0.9, stop=0.99),
],
regression=True,
y_tensor_type=T.imatrix,
objective_loss_function=binary_crossentropy,
#batch_iterator_train = BatchIterator(batch_size = 256),
max_epochs=30,
eval_size=0.0,
verbose=2,
)
seednumber = 1235
np.random.seed(seednumber)
net.fit(train, labels)
preds = net.predict_proba(test)[:, 0]
submission = pd.read_csv('../input/sample_submission.csv')
submission["TARGET"] = preds
submission.to_csv('Lasagne_bench.csv', index=False)
