def lasagne_oneLayer_classifier(param, X, labels):
## initialize the NN
layers0 = [('input', InputLayer),
('dense0', DenseLayer),
('dropout', DropoutLayer),
('output', DenseLayer)]
net0 = NeuralNet(layers=layers0,
input_shape=(None, param['num_features']),
dense0_num_units=param['dense0_num_units'],
dropout_p=param['dropout_p'],
output_num_units=param['num_classes'],
output_nonlinearity=softmax,
update=nesterov_momentum,
update_learning_rate=param['update_learning_rate'],
update_momentum=param['update_momentum'],
eval_size=0.02,
verbose=1,
max_epochs=param['max_epochs'])
## fit the results
net0.fit(X, labels)
return net0
def fit_nn_and_predict_probas(features, dv, features_t):
bwh = BestWeightsHolder()
tvs = TrainValidSplitter(standardize=True,few=True)
layers = [('input', InputLayer),
('dense0', DenseLayer),
('dropout0', DropoutLayer),
('dense1', DenseLayer),
('dropout1', DropoutLayer),
('output', DenseLayer)]
net = NeuralNet(layers=layers,
input_shape=(None, features.shape[1]),
dense0_num_units=512,
dropout0_p=0.4,
dense1_num_units=256,
dropout1_p=0.4,
output_num_units=38,
output_nonlinearity=softmax,
update=adagrad,
update_learning_rate=0.02,
train_split=tvs,
verbose=1,
max_epochs=40,
on_epoch_finished=[bwh.hold_best_weights])
holder = net.fit(features, dv)
holder.load_params_from(bwh.best_weights)
return holder.predict_proba(np.hstack((tvs.standa.transform(features_t[:,:23]), features_t[:,23:])))
def fit_model(train_x, y, test_x):
"""Feed forward neural network for kaggle digit recognizer competition.
Intentionally limit network size and optimization time (by choosing max_epochs = 15) to meet runtime restrictions
"""
print("\n\nRunning Convetional Net. Optimization progress below\n\n")
net1 = NeuralNet(
layers=[ #list the layers here
('input', layers.InputLayer),
('hidden1', layers.DenseLayer),
('output', layers.DenseLayer),
],
# layer parameters:
input_shape=(None, train_x.shape[1]),
hidden1_num_units=200, hidden1_nonlinearity=rectify, #params of first layer
output_nonlinearity=softmax, # softmax for classification problems
output_num_units=10, # 10 target values
# optimization method:
update=nesterov_momentum,
update_learning_rate=0.05,
update_momentum=0.7,
regression=False,
max_epochs=10, # Intentionally limited for execution speed
verbose=1,
)
net1.fit(train_x, y)
predictions = net1.predict(test_x)
return(predictions)
def test_diamond(self, NeuralNet):
input, hidden1, hidden2, concat, output = (
Mock(), Mock(), Mock(), Mock(), Mock())
nn = NeuralNet(
layers=[
('input', input),
('hidden1', hidden1),
('hidden2', hidden2),
('concat', concat),
('output', output),
],
input_shape=(10, 10),
hidden2_incoming='input',
concat_incoming=['hidden1', 'hidden2'],
)
nn.initialize_layers(nn.layers)
input.assert_called_with(name='input', shape=(10, 10))
hidden1.assert_called_with(incoming=input.return_value, name='hidden1')
hidden2.assert_called_with(incoming=input.return_value, name='hidden2')
concat.assert_called_with(
incoming=[hidden1.return_value, hidden2.return_value],
name='concat'
)
output.assert_called_with(incoming=concat.return_value, name='output')
def train():
weather = load_weather()
training = load_training()
X = assemble_X(training, weather)
print len(X[0])
mean, std = normalize(X)
y = assemble_y(training)
input_size = len(X[0])
learning_rate = theano.shared(np.float32(0.1))
net = NeuralNet(
layers=[
('input', InputLayer),
('hidden1', DenseLayer),
('dropout1', DropoutLayer),
('hidden2', DenseLayer),
('dropout2', DropoutLayer),
('output', DenseLayer),
],
# layer parameters:
input_shape=(None, input_size),
hidden1_num_units=325,
dropout1_p=0.4,
hidden2_num_units=325,
dropout2_p=0.4,
output_nonlinearity=sigmoid,
output_num_units=1,
# optimization method:
update=nesterov_momentum,
update_learning_rate=learning_rate,
update_momentum=0.9,
# Decay the learning rate
on_epoch_finished=[
AdjustVariable(learning_rate, target=0, half_life=1),
],
# This is silly, but we don't want a stratified K-Fold here
# To compensate we need to pass in the y_tensor_type and the loss.
regression=True,
y_tensor_type = T.imatrix,
objective_loss_function = binary_crossentropy,
max_epochs=85,
eval_size=0.1,
verbose=1,
)
X, y = shuffle(X, y, random_state=123)
net.fit(X, y)
_, X_valid, _, y_valid = net.train_test_split(X, y, net.eval_size)
probas = net.predict_proba(X_valid)[:,0]
print("ROC score", metrics.roc_auc_score(y_valid, probas))
return net, mean, std
def test_initialization(self, NeuralNet):
input, hidden1, hidden2, output = Mock(), Mock(), Mock(), Mock()
nn = NeuralNet(
layers=[
(input, {'shape': (10, 10), 'name': 'input'}),
(hidden1, {'some': 'param', 'another': 'param'}),
(hidden2, {}),
(output, {'name': 'output'}),
],
input_shape=(10, 10),
mock1_some='iwin',
)
out = nn.initialize_layers(nn.layers)
input.assert_called_with(
name='input', shape=(10, 10))
nn.layers_['input'] is input.return_value
hidden1.assert_called_with(
incoming=input.return_value, name='mock1',
some='iwin', another='param')
nn.layers_['mock1'] is hidden1.return_value
hidden2.assert_called_with(
incoming=hidden1.return_value, name='mock2')
nn.layers_['mock2'] is hidden2.return_value
output.assert_called_with(
incoming=hidden2.return_value, name='output')
assert out is nn.layers_['output']
def test_initialization_with_layer_instance(self, NeuralNet):
layer1 = InputLayer(shape=(128, 13)) # name will be assigned
layer2 = DenseLayer(layer1, name='output', num_units=2) # has name
nn = NeuralNet(layers=layer2)
out = nn.initialize_layers()
assert nn.layers_['output'] == layer2 == out[0]
assert nn.layers_['input0'] == layer1
def train(x_train, y_train):
clf_nn = NeuralNet(
layers=[ # three layers: one hidden layer
('input', layers.InputLayer),
('hidden1', layers.DenseLayer),
('hidden2', layers.DenseLayer),
('output', layers.DenseLayer),
],
# layer parameters:
input_shape=(None, 2538), # 784 input pixels per batch
hidden1_num_units=100, # number of units in hidden layer
hidden2_num_units=100,
output_nonlinearity=nonlinearities.softmax, # output layer uses identity function
output_num_units=10, # 10 target values
# optimization method:
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
max_epochs=50, # we want to train this many epochs
verbose=1,
)
clf_nn.fit(x_train, y_train)
return clf_nn
def test_initialization_legacy(self, NeuralNet):
input, hidden1, hidden2, output = Mock(), Mock(), Mock(), Mock()
nn = NeuralNet(
layers=[
('input', input),
('hidden1', hidden1),
('hidden2', hidden2),
('output', output),
],
input_shape=(10, 10),
hidden1_some='param',
)
out = nn.initialize_layers(nn.layers)
input.assert_called_with(
name='input', shape=(10, 10))
nn.layers_['input'] is input.return_value
hidden1.assert_called_with(
incoming=input.return_value, name='hidden1', some='param')
nn.layers_['hidden1'] is hidden1.return_value
hidden2.assert_called_with(
incoming=hidden1.return_value, name='hidden2')
nn.layers_['hidden2'] is hidden2.return_value
output.assert_called_with(
incoming=hidden2.return_value, name='output')
assert out is nn.layers_['output']
def test_initialization_with_tuples(self, NeuralNet):
input = Mock(__name__='InputLayer', __bases__=(InputLayer,))
hidden1, hidden2, output = [
Mock(__name__='MockLayer', __bases__=(Layer,)) for i in range(3)]
nn = NeuralNet(
layers=[
(input, {'shape': (10, 10), 'name': 'input'}),
(hidden1, {'some': 'param', 'another': 'param'}),
(hidden2, {}),
(output, {'name': 'output'}),
],
input_shape=(10, 10),
mock1_some='iwin',
)
out = nn.initialize_layers(nn.layers)
input.assert_called_with(
name='input', shape=(10, 10))
assert nn.layers_['input'] is input.return_value
hidden1.assert_called_with(
incoming=input.return_value, name='mock1',
some='iwin', another='param')
assert nn.layers_['mock1'] is hidden1.return_value
hidden2.assert_called_with(
incoming=hidden1.return_value, name='mock2')
assert nn.layers_['mock2'] is hidden2.return_value
output.assert_called_with(
incoming=hidden2.return_value, name='output')
assert out[0] is nn.layers_['output']
def test_clone():
from nolearn.lasagne import NeuralNet
from nolearn.lasagne import BatchIterator
from nolearn.lasagne import objective
params = dict(
layers=[
('input', InputLayer),
('hidden', DenseLayer),
('output', DenseLayer),
],
input_shape=(100, 784),
output_num_units=10,
output_nonlinearity=softmax,
more_params={
'hidden_num_units': 100,
},
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
regression=False,
objective=objective,
objective_loss_function=categorical_crossentropy,
batch_iterator_train=BatchIterator(batch_size=100),
y_tensor_type=T.ivector,
use_label_encoder=False,
on_epoch_finished=None,
on_training_finished=None,
max_epochs=100,
eval_size=0.1, # BBB
check_input=True,
verbose=0,
)
nn = NeuralNet(**params)
nn2 = clone(nn)
params1 = nn.get_params()
params2 = nn2.get_params()
for ignore in (
'batch_iterator_train',
'batch_iterator_test',
'output_nonlinearity',
'loss',
'objective',
'train_split',
'eval_size',
'X_tensor_type',
'on_epoch_finished',
'on_batch_finished',
'on_training_started',
'on_training_finished',
'custom_scores',
):
for par in (params, params1, params2):
par.pop(ignore, None)
assert params == params1 == params2
def train_net(X, y):
net2 = NeuralNet(
layers=[
('input', layers.InputLayer),
('ncaa', NCAALayer),
('dropout1', layers.DropoutLayer),
('hidden', layers.DenseLayer),
('dropout2', layers.DropoutLayer),
('output', layers.DenseLayer),
],
input_shape = (None, num_features * 2),
ncaa_num_units = 128,
dropout1_p=0.2,
hidden_num_units=128,
dropout2_p=0.3,
output_nonlinearity=nonlinearities.sigmoid,
output_num_units=1,
update=nesterov_momentum,
update_learning_rate=theano.shared(float32(0.01)),
update_momentum=theano.shared(float32(0.9)),
regression=True, # flag to indicate we're dealing with regression problem
max_epochs=20, # we want to train this many epochs
verbose=1,
)
net2.fit(X, y)
return net2
def fit(xTrain, yTrain, dense0_num=800, dropout_p=0.5, dense1_num=500, update_learning_rate=0.01,
update_momentum=0.9, test_ratio=0.2, max_epochs=20):
#update_momentum=0.9, test_ratio=0.2, max_epochs=20, train_fname='train.csv'):
#xTrain, yTrain, encoder, scaler = load_train_data(train_fname)
#xTest, ids = load_test_data('test.csv', scaler)
num_features = len(xTrain[0,:])
num_classes = 9
print num_features
layers0 = [('input', InputLayer),
('dense0', DenseLayer),
('dropout', DropoutLayer),
('dense1', DenseLayer),
('output', DenseLayer)]
clf = NeuralNet(layers=layers0,
input_shape=(None, num_features),
dense0_num_units=dense0_num,
dropout_p=dropout_p,
dense1_num_units=dense1_num,
output_num_units=num_classes,
output_nonlinearity=softmax,
update=nesterov_momentum,
update_learning_rate=update_learning_rate,
update_momentum=update_momentum,
eval_size=test_ratio,
verbose=1,
max_epochs=max_epochs)
clf.fit(xTrain, yTrain)
ll_train = metrics.log_loss(yTrain, clf.predict_proba(xTrain))
print ll_train
return clf
def nn_example(data):
net1 = NeuralNet(
layers=[('input', layers.InputLayer),
('hidden', layers.DenseLayer),
('output', layers.DenseLayer),
],
# layer parameters:
input_shape=(None, 28*28),
hidden_num_units=100, # number of units in 'hidden' layer
output_nonlinearity=lasagne.nonlinearities.softmax,
output_num_units=10, # 10 target values for the digits 0, 1, 2, ..., 9
# optimization method:
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
max_epochs=10,
verbose=1,
)
# Train the network
net1.fit(data['X_train'], data['y_train'])
# Try the network on new data
print("Feature vector (100-110): %s" % data['X_test'][0][100:110])
print("Label: %s" % str(data['y_test'][0]))
print("Predicted: %s" % str(net1.predict([data['X_test'][0]])))
def fit(self,tr,add_feat_tr):
## if trend exists, remove trend
if self.trend ==1:
trend = self.est_trend(tr)
tr = tr-np.asarray(trend)
layers0=[
## 2 layers with one hidden layer
(InputLayer, {'shape': (None,8,self.window_length)}),
(DenseLayer, {'num_units': 8*self.window_length}),
(DropoutLayer, {'p':0.3}),
(DenseLayer, {'num_units': 8*self.window_length/3}),
## the output layer
(DenseLayer, {'num_units': 1, 'nonlinearity': None}),
]
feats = build_feat(tr, add_feat_tr, window_length=self.window_length)
print feats.shape
feat_target = get_target(tr,window_length=self.window_length)
print feat_target.shape
net0 = NeuralNet(
layers=layers0,
max_epochs=400,
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
verbose=1,
regression=True,
)
net0.fit(feats[:-1],feat_target)
return net0,feats,feat_target
def createNet(X, Y, ln, loadFile = ""):
net1 = NeuralNet(
layers=[ # four layers: two hidden layers
('input', layers.InputLayer),
('hidden', layers.DenseLayer),
('hidden1', layers.DenseLayer),
('hidden2', layers.DenseLayer),
('hidden3', layers.DenseLayer),
('output', layers.DenseLayer),
],
# layer parameters: Best 400 400
input_shape=(None, numInputs), # 31 inputs
hidden_num_units=400, # number of units in hidden layer
hidden1_num_units=400,
hidden2_num_units=400,
hidden3_num_units=400,
output_nonlinearity=None, # output layer uses identity function
output_num_units=numOutputs, # 4 outputs
# optimization method:
update=nesterov_momentum,
update_learning_rate=ln,
update_momentum=0.9,
regression=True, # flag to indicate we're dealing with regression problem
max_epochs=1500, # we want to train this many epochs
verbose=1,
)
#if (loadFile != ""):
#net1.load_params_from(loadFile)
net1.max_epochs = 50
net1.update_learning_rate = ln;
return net1
def loadNet(netName):
if os.path.exists(netName):
net = pickle.load(open(netName, "rb"))
else:
net = NeuralNet(
layers=[ # three layers: one hidden layer
('input', layers.InputLayer),
('hidden', layers.DenseLayer),
('output', layers.DenseLayer),
],
# layer parameters:
input_shape=(None, 9216), # 96x96 input pixels per batch
hidden_num_units=100, # number of units in hidden layer
output_nonlinearity=None, # output layer uses identity function
output_num_units=30, # 30 target values
# optimization method:
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
regression=True, # flag to indicate we're dealing with regression problem
max_epochs=400, # we want to train this many epochs
verbose=1,
)
X, y = load()
net.fit(X, y)
print("X.shape == {}; X.min == {:.3f}; X.max == {:.3f}".format(X.shape, X.min(), X.max()))
print("y.shape == {}; y.min == {:.3f}; y.max == {:.3f}".format(y.shape, y.min(), y.max()))
pickle.dump(net, open(netName, 'wb'), -1)
return net
def net_fitted(self, NeuralNet, X_train, y_train):
nn = NeuralNet(
layers=[
('input', InputLayer),
('conv1', Conv2DLayer),
('conv2', Conv2DLayer),
('pool2', MaxPool2DLayer),
('output', DenseLayer),
],
input_shape=(None, 1, 28, 28),
output_num_units=10,
output_nonlinearity=softmax,
more_params=dict(
conv1_filter_size=(5, 5), conv1_num_filters=16,
conv2_filter_size=(3, 3), conv2_num_filters=16,
pool2_pool_size=(8, 8),
hidden1_num_units=16,
),
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
max_epochs=3,
)
return nn.fit(X_train, y_train)
def train(self, X, y_train, X_test, ids_test, y_test, outfile, is_valid):
X = np.array(X)
encoder = LabelEncoder()
y = encoder.fit_transform(y_train).astype(np.int32)
num_classes = len(encoder.classes_)
num_features = X.shape[1]
layers0 = [('input', InputLayer),
('dense1', DenseLayer),
('dropout1', DropoutLayer),
('dense2', DenseLayer),
('dropout2', DropoutLayer),
('output', DenseLayer)]
net0 = NeuralNet(layers=layers0,
input_shape=(None, num_features),
dense1_num_units=3500,
dropout1_p=0.4,
dense2_num_units=2300,
dropout2_p=0.5,
output_num_units=num_classes,
output_nonlinearity=softmax,
#update=nesterov_momentum,
update=adagrad,
update_learning_rate=0.01,
#update_momentum=0.9,
#objective_loss_function=softmax,
objective_loss_function=categorical_crossentropy,
eval_size=0.2,
verbose=1,
max_epochs=20)
net0.fit(X, y)
X_test = np.array(X_test)
self.make_submission(net0, X_test, ids_test, encoder)
def test_initialization_with_tuples(self, NeuralNet):
input = Mock(__name__="InputLayer", __bases__=(InputLayer,))
hidden1, hidden2, output = [Mock(__name__="MockLayer", __bases__=(Layer,)) for i in range(3)]
nn = NeuralNet(
layers=[
(input, {"shape": (10, 10), "name": "input"}),
(hidden1, {"some": "param", "another": "param"}),
(hidden2, {}),
(output, {"name": "output"}),
],
input_shape=(10, 10),
mock1_some="iwin",
)
out = nn.initialize_layers(nn.layers)
input.assert_called_with(name="input", shape=(10, 10))
assert nn.layers_["input"] is input.return_value
hidden1.assert_called_with(incoming=input.return_value, name="mock1", some="iwin", another="param")
assert nn.layers_["mock1"] is hidden1.return_value
hidden2.assert_called_with(incoming=hidden1.return_value, name="mock2")
assert nn.layers_["mock2"] is hidden2.return_value
output.assert_called_with(incoming=hidden2.return_value, name="output")
assert out is nn.layers_["output"]
def test_diamond(self, NeuralNet):
input = Mock(__name__='InputLayer', __bases__=(InputLayer,))
hidden1, hidden2, concat, output = [
Mock(__name__='MockLayer', __bases__=(Layer,)) for i in range(4)]
nn = NeuralNet(
layers=[
('input', input),
('hidden1', hidden1),
('hidden2', hidden2),
('concat', concat),
('output', output),
],
input_shape=(10, 10),
hidden2_incoming='input',
concat_incomings=['hidden1', 'hidden2'],
)
nn.initialize_layers(nn.layers)
input.assert_called_with(name='input', shape=(10, 10))
hidden1.assert_called_with(incoming=input.return_value, name='hidden1')
hidden2.assert_called_with(incoming=input.return_value, name='hidden2')
concat.assert_called_with(
incomings=[hidden1.return_value, hidden2.return_value],
name='concat'
)
output.assert_called_with(incoming=concat.return_value, name='output')
def CompileNetwork(l_out, epochs, update, update_learning_rate, objective_l2,
earlystopping, patience, batch_size, verbose):
update_fn = getattr(updates, update)
earlystop = EarlyStopping(patience=patience, verbose=verbose)
net = NeuralNet(
l_out,
max_epochs=epochs,
update=update_fn,
objective_l2=objective_l2,
batch_iterator_train = BatchIterator(batch_size=batch_size),
batch_iterator_test = BatchIterator(batch_size=batch_size),
verbose=verbose,
on_training_finished = [earlystop.load_best_weights]
)
if earlystopping == True:
net.on_epoch_finished.append(earlystop)
if update_learning_rate is not None:
net.update_learning_rate=update_learning_rate
return net
def classif_no_valid(self, NeuralNet, X, y):
from nolearn.lasagne import TrainSplit
l = InputLayer(shape=(None, X.shape[1]))
l = DenseLayer(l, num_units=len(np.unique(y)), nonlinearity=softmax)
net = NeuralNet(
l, update_learning_rate=0.01, train_split=TrainSplit(0))
return net.fit(X, y)
def NN(X,y):
net1 = NeuralNet(
layers=[ # three layers: one hidden layer
('input', layers.InputLayer),
('hidden', layers.DenseLayer),
('output', layers.DenseLayer),
],
# layer parameters:
input_shape=(None, 9216), # 96x96 input pixels per batch
hidden_num_units=100, # number of units in hidden layer
output_nonlinearity=None, # output layer uses identity function
output_num_units=30, # 30 target values
# optimization method:
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
regression=True, # flag to indicate we're dealing with regression problem
max_epochs=400, # we want to train this many epochs
verbose=1,
)
net1.fit(X, y)
def neural_network(x_train, y_train):
X, y, encoder, scaler = load_train_data(x_train, y_train)
num_classes = len(encoder.classes_)
num_features = X.shape[1]
layers0 = [
("input", InputLayer),
("dropoutf", DropoutLayer),
("dense0", DenseLayer),
("dropout", DropoutLayer),
("dense1", DenseLayer),
("dropout2", DropoutLayer),
("output", DenseLayer),
]
net0 = NeuralNet(
layers=layers0,
input_shape=(None, num_features),
dropoutf_p=0.15,
dense0_num_units=1000,
dropout_p=0.25,
dense1_num_units=500,
dropout2_p=0.25,
output_num_units=num_classes,
output_nonlinearity=softmax,
update=adagrad,
update_learning_rate=0.005,
eval_size=0.01,
verbose=1,
max_epochs=30,
)
net0.fit(X, y)
return (net0, scaler)
def train_network():
layers0 = [('input', InputLayer),
('dense0', DenseLayer),
('dropout0', DropoutLayer),
('dense1', DenseLayer),
('dropout1', DropoutLayer),
('dense2', DenseLayer),
('output', DenseLayer)]
es = EarlyStopping(patience=200)
net0 = NeuralNet(layers=layers0,
input_shape=(None, num_features),
dense0_num_units=256,
dropout0_p=0.5,
dense1_num_units=128,
dropout1_p=0.5,
dense2_num_units=64,
output_num_units=num_classes,
output_nonlinearity=softmax,
update=nesterov_momentum,
update_learning_rate=theano.shared(float32(0.01)),
update_momentum=theano.shared(float32(0.9)),
eval_size=0.2,
verbose=1,
max_epochs=1000,
on_epoch_finished=[
AdjustVariable('update_learning_rate', start=0.01, stop=0.0001),
AdjustVariable('update_momentum', start=0.9, stop=0.999),
es
])
net0.fit(X, y)
return (es.best_valid, net0)
def test_initialization_legacy(self, NeuralNet):
input = Mock(__name__='InputLayer', __bases__=(InputLayer,))
hidden1, hidden2, output = [
Mock(__name__='MockLayer', __bases__=(Layer,)) for i in range(3)]
nn = NeuralNet(
layers=[
('input', input),
('hidden1', hidden1),
('hidden2', hidden2),
('output', output),
],
input_shape=(10, 10),
hidden1_some='param',
)
out = nn.initialize_layers(nn.layers)
input.assert_called_with(
name='input', shape=(10, 10))
assert nn.layers_['input'] is input.return_value
hidden1.assert_called_with(
incoming=input.return_value, name='hidden1', some='param')
assert nn.layers_['hidden1'] is hidden1.return_value
hidden2.assert_called_with(
incoming=hidden1.return_value, name='hidden2')
assert nn.layers_['hidden2'] is hidden2.return_value
output.assert_called_with(
incoming=hidden2.return_value, name='output')
assert out[0] is nn.layers_['output']
def _create_nnet(self, input_dims, output_dims, learning_rate, num_hidden_units=15, batch_size=32, max_train_epochs=1,
hidden_nonlinearity=nonlinearities.rectify, output_nonlinearity=None, update_method=updates.sgd):
"""
A subclass may override this if a different sort
of network is desired.
"""
nnlayers = [('input', layers.InputLayer), ('hidden', layers.DenseLayer), ('output', layers.DenseLayer)]
nnet = NeuralNet(layers=nnlayers,
# layer parameters:
input_shape=(None, input_dims),
hidden_num_units=num_hidden_units,
hidden_nonlinearity=hidden_nonlinearity,
output_nonlinearity=output_nonlinearity,
output_num_units=output_dims,
# optimization method:
update=update_method,
update_learning_rate=learning_rate,
regression=True, # flag to indicate we're dealing with regression problem
max_epochs=max_train_epochs,
batch_iterator_train=BatchIterator(batch_size=batch_size),
train_split=nolearn.lasagne.TrainSplit(eval_size=0),
verbose=0,
)
nnet.initialize()
return nnet
def build_mlp(input_var=None):
net1 = NeuralNet(
layers=[ # three layers: one hidden layer
('input', layers.InputLayer),
('hidden1', layers.DenseLayer),
('hidden2', layers.DenseLayer),
('output', layers.DenseLayer),
],
# layer parameters:
input_shape=(None, 14, 2177), # 14 x 2177 input pixels per batch
hidden1_num_units=100, # number of units in hidden layer
hidden2_num_units=100,
output_nonlinearity=lasagne.nonlinearities.softmax, # output layer uses identity function
output_num_units=2, # 2 target values
# optimization method:
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
#regression=False, # flag to indicate we're dealing with regression problem
max_epochs=500, # we want to train this many epochs
verbose=1,
)
X, y = load_dataset()
y = np.asanyarray(y,np.int32)
print(X.shape)
print(y.shape)
net1.fit(X, y)
class NN(object):
def __init__(self, input_size, hidden_1_size, hidden_2_size=None):
n_layers = [
('input', layers.InputLayer),
('hidden1', layers.DenseLayer),
('dropout1', layers.DropoutLayer)
]
if hidden_2_size is not None:
n_layers.extend(
[('hidden2', layers.DenseLayer), ('dropout2', layers.DropoutLayer)]
)
n_layers.append(('output', layers.DenseLayer))
self.model = NeuralNet(
layers=n_layers,
input_shape=(None, input_size),
hidden1_num_units=hidden_1_size, dropout1_p=0.5,
output_nonlinearity=tanh,
output_num_units=1,
regression=True,
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
eval_size=0.1,
on_epoch_finished=[
AdjustVariable('update_learning_rate', stop=0.0001, decrement=0.00001),
AdjustVariable('update_momentum', stop=0.999, increment=0.0001),
EarlyStopping(patience=100)
],
max_epochs=5000,
verbose=1
)
if hidden_2_size is not None:
self.model.__dict__['hidden2_num_units'] = hidden_2_size
self.model.__dict__['dropout2_p'] = 0.5
def train(self, X, Y):
self.model.fit(np.asarray(X, dtype=np.float32), np.asarray(Y, dtype=np.float32))
def predict_continuous(self, X_test):
return self.model.predict(np.asarray(X_test, dtype=np.float32))
def predict_classes(self, X_test):
Y_pred = self.predict_continuous(X_test)
# threshold the continuous values to get the classes
pos = Y_pred >= .33
neg = Y_pred <= -0.33
neu = np.logical_and(Y_pred < 0.33, Y_pred > -0.33)
Y_pred[pos] = 1
Y_pred[neg] = -1
Y_pred[neu] = 0
return Y_pred.reshape(-1)
num_classes = len(encoder.classes_)
num_features = X.shape[1]
layers0 = [('input', InputLayer),
('dense0', DenseLayer),
('dropout', DropoutLayer),
('dense1', DenseLayer),
('output', DenseLayer)]
net0 = NeuralNet(layers=layers0,
input_shape=(None, num_features),
dense0_num_units=200,
dropout_p=0.5,
dense1_num_units=200,
output_num_units=num_classes,
output_nonlinearity=softmax,
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
eval_size=0.2,
verbose=1,
max_epochs=20)
net0.fit(X, y)
make_submission(net0, X_test, ids, encoder)
def main(input_file, model_path):
batch_size = 128
nb_classes = 62 # A-Z, a-z and 0-9
nb_epoch = 2
# Input image dimensions
img_rows, img_cols = 32, 32
# Path of data files
path = input_file
### PREDICTION ###
# Load the model with the highest validation accuracy
# model.load_weights("best.kerasModelWeights")
# Load Kaggle test set
X_test = np.load(path + "/testPreproc_" + str(img_rows) + "_" +
str(img_cols) + ".npy")
print X_test.shape
# Load the preprocessed data and labels
X_train_all = np.load(path + "/trainPreproc_" + str(img_rows) + "_" +
str(img_cols) + ".npy")
Y_train_all = np.load(path + "/labelsPreproc.npy")
X_train, X_val, Y_train, Y_val = \
train_test_split(X_train_all, Y_train_all, test_size=0.25, stratify=np.argmax(Y_train_all, axis=1))
print X_train.shape
Y_val = convert_(Y_val)
X_train = X_train.reshape((-1, 1, 32, 32))
#
# # input shape for neural network
# labels = labels.astype(np.uint8)
X_val = X_val.reshape((-1, 1, 32, 32))
#
# # input shape for neural network
Y_val = Y_val.astype(np.uint8)
#
input_image_vector_shape = (None, 1, 32, 32)
net1 = NeuralNet(
layers=[
('input', layers.InputLayer),
('conv2d1', layers.Conv2DLayer),
('maxpool1', layers.MaxPool2DLayer),
('conv2d2', layers.Conv2DLayer),
('maxpool2', layers.MaxPool2DLayer),
('conv2d3', layers.Conv2DLayer),
('maxpool3', layers.MaxPool2DLayer),
# ('conv2d4', layers.Conv2DLayer),
# ('maxpool4', layers.MaxPool2DLayer),
('dropout1', layers.DropoutLayer),
('dropout2', layers.DropoutLayer),
('dense', layers.DenseLayer),
# ('dense2', layers.DenseLayer),
('output', layers.DenseLayer),
],
input_shape=input_image_vector_shape,
conv2d1_num_filters=128,
conv2d1_filter_size=(3, 3),
conv2d1_nonlinearity=lasagne.nonlinearities.tanh,
conv2d1_W=lasagne.init.GlorotUniform(),
conv2d1_pad=(2, 2),
maxpool1_pool_size=(2, 2),
conv2d2_num_filters=256,
conv2d2_filter_size=(3, 3),
conv2d2_nonlinearity=lasagne.nonlinearities.rectify,
conv2d2_pad=(2, 2),
maxpool2_pool_size=(2, 2),
conv2d3_num_filters=512,
conv2d3_filter_size=(3, 3),
conv2d3_nonlinearity=lasagne.nonlinearities.rectify,
conv2d3_pad=(2, 2),
maxpool3_pool_size=(2, 2),
dropout1_p=0.5,
dropout2_p=0.5,
dense_num_units=8192,
dense_nonlinearity=lasagne.nonlinearities.rectify,
# dense2_num_units = 16,
# dense2_nonlinearity = lasagne.nonlinearities.rectify,
output_nonlinearity=lasagne.nonlinearities.softmax,
output_num_units=62,
update=momentum,
# 75.5 with tanh init dense num = 256%
update_learning_rate=0.03,
update_momentum=0.8,
max_epochs=1000,
verbose=1,
)
print "Loading Neural Net Parameters..."
net1.initialize_layers()
net1.load_weights_from('{}_weightfile.w'.format(model_path))
net1.load_params_from('{}_paramfile.w'.format(model_path))
from sklearn.metrics import classification_report, accuracy_score, confusion_matrix
print 'Testing...'
y_true, y_pred = Y_val, net1.predict(X_val) # Get our predictions
print(classification_report(y_true,
y_pred)) # Classification on each digit
print net1.predict(X_val)
print Y_val
a = confusion_matrix(Y_val, net1.predict(X_val))
b = np.trace(a)
print 'Training Accuracy: ' + str(float(b) / float(np.sum(a)))
(DenseLayer, {
'num_units': 128
}),
# the output layer
(DenseLayer, {
'num_units': 10,
'nonlinearity': softmax
}),
]
net0 = NeuralNet(
layers=layers0,
max_epochs=50,
update=adam,
update_learning_rate=0.0002,
# update_momentum=0.9,
objective_l2=0.0025,
train_split=TrainSplit(eval_size=0.1),
verbose=2,
)
# net0.initialize()
# layer_info = PrintLayerInfo()
# layer_info(net0)
net0.fit(X_train, y_train)
result = net0.predict(X_test)
np.save('cnn_res.npy', result)
#fName = "./features/cnn.pkl"
#print("Saving CNN to ./features/cnn.pkl")
#with open(fName, "w") as f:
def main():
print(" - Start.")
t0 = time()
trainpath = "../data/train.csv"
testpath = "../data/test.csv"
subpath = "sub.csv"
X_all, y_all, ids_all = load_shuf_all_data_ids(trainpath)
# X_all = X_all[:20000]
# y_all = y_all[:20000]
encoder = skl_pre.LabelEncoder()
y_all = encoder.fit_transform(y_all).astype(np.int32)
print 'X_all:', X_all
print 'y_all:', y_all
# new features
# f1=True ; f2=True ; f3=True ; f4=True ; str_opt = '_f1234'
f1 = False
f2 = False
f3 = False
f4 = False
str_opt = '_'
X_all = F_addnewfeats(X_all, f1, f2, f3, f4)
# to log scale
LOG = True
if LOG:
print '\n log(1+x) to all features...'
X_all = np.log1p(X_all)
str_opt = str_opt + 'l'
# scaling
SCA = True
if SCA:
print '\n Scaling features...'
scaler = skl_pre.StandardScaler().fit(X_all)
X_all = scaler.transform(X_all)
str_opt = str_opt + 'st'
# PCA
PCA = False
if PCA:
print '\n PCA...'
pca = skl_dec.PCA(n_components='mle').fit(X_all)
print(' ... num components: %i , variance retained: %.2f' %
(len(pca.components_), sum(pca.explained_variance_ratio_)))
X_all = pca.transform(X_all)
print '\n X_all[0]:', X_all[0]
str_opt = str_opt + ('p%.2f' % sum(pca.explained_variance_ratio_))
# Prepare neural network:
num_classes = len(encoder.classes_)
num_features = X_all.shape[1]
layers0 = [
('input', las_lay.InputLayer),
('dropout0', las_lay.DropoutLayer),
('hidden1', las_lay.DenseLayer),
('dropout1', las_lay.DropoutLayer),
('hidden2', las_lay.DenseLayer),
('dropout2', las_lay.DropoutLayer),
# ('hidden3', las_lay.DenseLayer),
('output', las_lay.DenseLayer)
]
NNargs = dict(
layers=layers0,
input_shape=(None, num_features),
dropout0_p=0.15,
hidden1_num_units=1000,
dropout1_p=0.25,
hidden2_num_units=500,
dropout2_p=0.25,
# hidden3_num_units=150,
# hidden*_nonlinearity=rectifier by default, try softmax
output_num_units=num_classes,
output_nonlinearity=softmax,
#
update=adagrad, # nesterov_momentum, # adagrad, rmsprop
update_learning_rate=theano.shared(float32(0.03)), # 0.01,
# update_momentum=theano.shared(float32(0.9)), # 0.9, ONLY USE WITH nesterov_momentum
on_epoch_finished=[
AdjustVariable('update_learning_rate', start=0.03, stop=0.0001),
# AdjustVariable('update_momentum', start=0.9, stop=0.999), # ONLY USE WITH nesterov_momentum
EarlyStopping(patience=10),
],
#
eval_size=0.2, # this fraction of the set is used for validation (test)
verbose=1,
max_epochs=150) #,
# objective = MyObjective)
global GLOBrealnumepochs
GLOBrealnumepochs = NNargs["max_epochs"]
clf = NeuralNet(**NNargs)
multiclass_log_loss = skl_met.make_scorer(score_func=logloss_mc,
greater_is_better=False,
needs_proba=True)
num_cores = multiprocessing.cpu_count()
# single run:
SINGLE = False
if SINGLE:
clf.fit(X_all, y_all)
train_loss = np.array([i["train_loss"] for i in clf.train_history_])
valid_loss = np.array([i["valid_loss"] for i in clf.train_history_])
pyplot.plot(train_loss, linewidth=3, label="train")
pyplot.plot(valid_loss, linewidth=3, label="valid")
pyplot.grid()
pyplot.legend()
pyplot.xlabel("epoch")
pyplot.ylabel("loss")
pyplot.ylim(0.4, 0.8)
pyplot.yscale("log")
# pyplot.show()
pyplot.savefig('learningcurves.png')
# cv score on 1 classifier
CV = False
if CV:
scores = skl_cv.cross_val_score(clf,
X_all,
y_all,
cv=3,
scoring=multiclass_log_loss,
n_jobs=3)
print('\n\n ... done in %.0fs\n\n' % (time() - t0))
print '\n cv scores of 1 classifier:', scores
print '\n mean score=', np.mean(scores)
sys.stdout.flush()
# grid
GRID = False
if GRID:
print '\n\n ** GRID SEARCH ** \n\n'
grid_clf = skl_grid.GridSearchCV(
clf,
param_grid={
'hidden1_num_units':
[3000, 4000], #[100, 150, 200, 250, 300, 400, 600],
'dropout1_p': [0.6, 0.8], #, 0.5, 0.7],
'hidden2_num_units': [300, 500]
},
scoring=multiclass_log_loss,
cv=3,
n_jobs=3)
print(' ... grid created')
print('\n Fitting grid...')
tfit = time()
sys.stdout.flush()
grid_clf.fit(X_all, y_all)
print(' ... grid fitted in %.2fs' % (time() - tfit))
print("\n Grid scores:")
for params, mean_score, scores in grid_clf.grid_scores_:
print("%0.5f (+/-%0.03f) for %r" %
(mean_score, scores.std() / 2, params))
print("\n Best estimator:")
print(grid_clf.best_estimator_)
print("\n Best params:")
print(grid_clf.best_params_)
print("\n Best score:")
print(grid_clf.best_score_)
CV4ENS = True
if CV4ENS:
# writing the predictions of 5 CV in files for future testing of ensemble
n_folds = 5
kf = skl_cv.KFold(len(X_all), n_folds=n_folds, random_state=13)
scores = []
ids_total = np.array([])
y_prob_total = np.empty((1, num_classes))
for train, test in kf:
X_train, X_test, y_train, y_test = X_all[train], X_all[
test], y_all[train], y_all[test]
ids_total = np.append(ids_total, ids_all[test])
BAG = True
if BAG:
print(
"\n\n Fit and predict for different realizations of same architecture (diff seeds)"
)
num_bags = 5
best_max_epochs = {}
for i in xrange(num_bags):
best_max_epochs[i] = 0
# best_max_epochs = { 0: 1, 1:1, 2:1, 3:1, 4:1} #68, 1: 57, 2: 46} # zeros for bags you already dont know best n_epochs
assert len(best_max_epochs) == num_bags
probs_bags = Parallel(n_jobs=num_cores)(
delayed(calc_prob_bag)(i, best_max_epochs[i], NNargs,
X_all, y_all, X_test)
for i in xrange(num_bags))
y_prob = sum(probs_bags) / num_bags
print 'y_prob:', np.shape(y_prob), y_prob
y_prob_total = np.concatenate((y_prob_total, y_prob), axis=0)
else:
clf = NeuralNet(**NNargs)
clf.fit(X_train, y_train)
y_prob = clf.predict_proba(X_test)
y_prob_total = np.concatenate((y_prob_total, y_prob), axis=0)
y_prob_total = np.delete(y_prob_total, 0,
0) # removes first row (created by np.empty)
# subfile = 'LNN4' + str_opt + '_e%i_h%i_d%.1f_h%i_CVALL.csv' % (NNargs["max_epochs"],NNargs["hidden1_num_units"],
# NNargs["dropout1_p"], NNargs["hidden2_num_units"])
subfile = 'LNN4sbag%i%s_d%.2f_h%i_d%.2f_h%i_d%.2f_CVALL.csv' % (
num_bags, str_opt, NNargs["dropout0_p"],
NNargs["hidden1_num_units"], NNargs["dropout1_p"],
NNargs["hidden2_num_units"], NNargs["dropout2_p"])
# subfile = 'LNN5' + str_opt + '_e%i_h%i_d%.1f_h%i_d%.1f_h%i_CVALL.csv' % (NNargs["max_epochs"], NNargs["hidden1_num_units"],
# NNargs["dropout1_p"], NNargs["hidden2_num_units"],
# NNargs["dropout2_p"], NNargs["hidden3_num_units"])
write_submission(ids_total, encoder, y_prob_total, path=subfile)
SUB = False
if SUB:
print("\n\n Starting submission process...")
# encoder = skl_pre.LabelEncoder()
# y_true = encoder.fit_transform(y_all)
# assert (encoder.classes_ == clf_final.classes_).all()
X_test, ids = load_test_data(path=testpath)
X_test = F_addnewfeats(X_test, f1, f2, f3, f4)
if LOG: X_test = np.log1p(X_test)
if SCA: X_test = scaler.transform(X_test)
if PCA: X_test = pca.transform(X_test)
if GRID:
print('\n Setting NN params to best values in the grid...')
NNargs["hidden1_num_units"] = grid_clf.best_params_[
'hidden1_num_units']
NNargs["dropout1_p"] = grid_clf.best_params_['dropout1_p']
NNargs["hidden2_num_units"] = grid_clf.best_params_[
'hidden2_num_units']
RECALCEPOC = False
if RECALCEPOC:
print '\n Refitting to get the num epochs optimal (with smaller eval_size and more patience)...'
NNargs[
"eval_size"] = 0.05 # just a small test set to derive optimal numepochs, closer to final training with all set
if len(NNargs["on_epoch_finished"]) == 3:
NNargs["on_epoch_finished"][-1] = EarlyStopping(
patience=25) # more patience for final sub
clf_final = NeuralNet(**NNargs)
clf_final.fit(X_all, y_all)
print ' ... done refitting to obtain GLOBrealnumepochs:', GLOBrealnumepochs
BAG = True
if BAG:
# see https://www.kaggle.com/c/otto-group-product-classification-challenge/forums/t/13851/lasagne-with-2-hidden-layers
print(
"\n\n Fit and predict for different realiztions of same architecture (diff seeds)"
)
print(
"\n\n Fit and predict for different realizations of same architecture (diff seeds)"
)
num_bags = 5
best_max_epochs = {
0: 0,
1: 0,
2: 0,
3: 0,
4: 0,
5: 0
} # { 0: 68, 1: 57, 2: 46} # zeros for bags you already dont know best n_epochs
assert len(best_max_epochs) == num_bags
probs_bags = Parallel(n_jobs=num_cores)(delayed(calc_prob_bag)(
i, best_max_epochs[i], NNargs, X_all, y_all, X_test)
for i in xrange(num_bags))
probs_final = sum(probs_bags) / num_bags
subfile = 'LNN4bag%i%s_d%.2f_h%i_d%.2f_h%i_d%.2f.csv' % (
num_bags, str_opt, NNargs["dropout0_p"],
NNargs["hidden1_num_units"], NNargs["dropout1_p"],
NNargs["hidden2_num_units"], NNargs["dropout2_p"])
print("\n writing submission to file: %s" % subfile)
write_submission(ids, encoder, probs_final, path=subfile)
sys.exit()
# re-set properties to train with all set
NNargs["eval_size"] = 0.0001
if len(NNargs["on_epoch_finished"]) == 3:
del NNargs["on_epoch_finished"][-1] # removes the EarlyStopping
NNargs["max_epochs"] = GLOBrealnumepochs
clf_final = NeuralNet(**NNargs)
print("\n Writing submission file for best estimator")
# subfile = 'LNN4' + str_opt + '_e%i_h%i_d%.1f_h%i.csv' % (NNargs["max_epochs"], NNargs["hidden1_num_units"],
# NNargs["dropout1_p"], NNargs["hidden2_num_units"])
subfile = 'LNN5' + str_opt + '_e%i_h%i_d%.1f_h%i_d%.1f_h%i.csv' % (
NNargs["max_epochs"], NNargs["hidden1_num_units"],
NNargs["dropout1_p"], NNargs["hidden2_num_units"],
NNargs["dropout2_p"], NNargs["hidden3_num_units"])
print '\n name:', subfile, '\n'
print(" re-fitting with all training set...")
clf_final.fit(X_all, y_all)
make_submission(X_test, ids, clf_final, encoder, path=subfile)
(DropoutLayer, {'p': 0.5}),
(DenseLayer, {'num_units': 512}),
# the output layer
(DropoutLayer, {'p': 0.5}),
(DenseLayer, {'num_units': 2, 'nonlinearity': softmax}),
]
# Network parameters
net0 = NeuralNet(
layers=layers0,
max_epochs=100,
batch_iterator_train = BatchIterator(batch_size=100, shuffle=True),
update=nesterov_momentum,
update_learning_rate=0.01,
objective_l2=0.001,
train_split=TrainSplit(eval_size=0.2),
verbose=2,
)
# Train
net0.fit(x_train, y_train)
# Plot learning curve
plot_loss(net0)
# Plot learned filters
#plot_conv_weights(net0.layers_[1], figsize=(4, 4)) # Layer 1 (conv1)
net1 = NeuralNet(
layers=[("input", layers.InputLayer), ("conv2d1", layers.Conv2DLayer),
("maxpool1", layers.MaxPool2DLayer),
("conv2d2", layers.Conv2DLayer),
("maxpool2", layers.MaxPool2DLayer),
("dropout1", layers.DropoutLayer), ("dense", layers.DenseLayer),
("dropout2", layers.DropoutLayer), ("output", layers.DenseLayer)],
# input layer
input_shape=(None, 1, 28, 28),
# layer conv2d1
conv2d1_num_filters=32,
conv2d1_filter_size=(5, 5),
conv2d1_nonlinearity=lasagne.nonlinearities.rectify,
conv2d1_W=lasagne.init.GlorotUniform(),
# layer maxpool1
maxpool1_pool_size=(2, 2),
# layer conv2d2
conv2d2_num_filters=32,
conv2d2_filter_size=(5, 5),
conv2d2_nonlinearity=lasagne.nonlinearities.rectify,
# layer maxpool2
maxpool2_pool_size=(2, 2),
# dropout1
dropout1_p=0.5,
# dense
dense_num_units=256,
dense_nonlinearity=lasagne.nonlinearities.rectify,
# dropout2
dropout2_p=0.5,
# output
output_nonlinearity=lasagne.nonlinearities.softmax,
output_num_units=10,
# optimization method params
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
max_epochs=10,
verbose=1,
)
else:
y = None
return X, y
net1 = NeuralNet(
layers=[ # three layers: one hidden layer
('input', layers.InputLayer),
('hidden', layers.DenseLayer),
('output', layers.DenseLayer),
],
# layer parameters:
input_shape=(None, 9216), # 96x96 input pixels per batch
hidden_num_units=100, # number of units in hidden layer
output_nonlinearity=None, # output layer uses identity function
output_num_units=30, # output layer has 30 units (target values)
# optimization method:
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
regression=True, # flag to indicate we're dealing with a regression problem
max_epochs=400, # we want to train this many epochs
verbose=1,
)
X, y = load()
print("X.shape == {}; X.min == {:.3f}; X.max == {:.3f}".format(
X.shape, X.min(), X.max()))
print("y.shape == {}; y.min == {:.3f}; y.max == {:.3f}".format(
y.shape, y.min(), y.max()))
def test_legacy_eval_size(self, NeuralNet):
net = NeuralNet([], eval_size=0.3, max_epochs=0)
assert net.train_split.eval_size == 0.3
def test_initialization_with_layer_instance_bad_params(self, NeuralNet):
layer = DenseLayer(InputLayer(shape=(128, 13)), num_units=2)
nn = NeuralNet(layers=layer, dense1_num_units=3)
with pytest.raises(ValueError):
nn.initialize_layers()
net = NeuralNet(
layers=[
('input', layers.InputLayer),
('conv2d1', layers.Conv2DLayer),
('maxpool1', layers.MaxPool2DLayer),
# ('dropout1', layers.DropoutLayer),
('conv2d2', layers.Conv2DLayer),
('maxpool2', layers.MaxPool2DLayer),
('conv2d3', layers.Conv2DLayer),
('maxpool3', layers.MaxPool2DLayer),
('conv2d4', layers.Conv2DLayer),
('maxpool4', layers.MaxPool2DLayer),
('dense1', layers.DenseLayer),
('dropout2', layers.DropoutLayer),
('output', layers.DenseLayer)
],
# input layer descriptors
input_shape=(None, 1, PIXELS, PIXELS),
# convolution layer descriptors
conv2d1_num_filters=16,
conv2d1_filter_size=(5, 5),
conv2d1_nonlinearity=lasagne.nonlinearities.rectify,
conv2d1_W=lasagne.init.GlorotUniform(),
# maxppol layer descriptors
maxpool1_pool_size=(2, 2),
# dropout
# dropout1_p = 0.5,
# convolution layer descriptors
conv2d2_num_filters=32,
conv2d2_filter_size=(3, 3),
conv2d2_nonlinearity=lasagne.nonlinearities.rectify,
conv2d2_W=lasagne.init.GlorotUniform(),
# maxppol layer descriptors
maxpool2_pool_size=(2, 2),
# convolution layer descriptors
conv2d3_num_filters=64,
conv2d3_filter_size=(3, 3),
conv2d3_nonlinearity=lasagne.nonlinearities.rectify,
conv2d3_W=lasagne.init.GlorotUniform(),
maxpool3_pool_size=(2, 2),
# convolution layer descriptors
conv2d4_num_filters=128,
conv2d4_filter_size=(3, 3),
conv2d4_nonlinearity=lasagne.nonlinearities.rectify,
conv2d4_W=lasagne.init.GlorotUniform(),
# maxppol layer descriptors
maxpool4_pool_size=(2, 2),
dense1_num_units=128,
# dropout layer descriptors
dropout2_p=0.5,
# output layer descriptors
output_nonlinearity=lasagne.nonlinearities.softmax,
output_num_units=10,
#optimization parameters
update=nesterov_momentum,
update_learning_rate=0.01,
max_epochs=max_epochs,
verbose=1000000)
def train():
weather = load_weather()
training = load_training()
X = assemble_X(training, weather)
mean, std = normalize(X)
y = assemble_y(training)
input_size = len(X[0])
learning_rate = theano.shared(np.float32(0.1))
net = NeuralNet(
layers=[
('input', InputLayer),
('hidden1', DenseLayer),
('dropout1', DropoutLayer),
('hidden2', DenseLayer),
('dropout2', DropoutLayer),
('hidden3', DenseLayer),
('dropout3', DropoutLayer),
('output', DenseLayer),
],
# layer parameters:
input_shape=(None, input_size),
hidden1_num_units=325,
dropout1_p=0.4,
hidden2_num_units=325,
dropout2_p=0.4,
hidden3_num_units=160,
dropout3_p=0.4,
output_nonlinearity=sigmoid,
output_num_units=1,
# optimization method:
update=nesterov_momentum,
update_learning_rate=learning_rate,
update_momentum=0.9,
# Decay the learning rate
on_epoch_finished=[
AdjustVariable(learning_rate, target=0, half_life=4),
],
# This is silly, but we don't want a stratified K-Fold here
# To compensate we need to pass in the y_tensor_type and the loss.
regression=True,
y_tensor_type=T.imatrix,
objective_loss_function=binary_crossentropy,
max_epochs=75,
eval_size=0.1,
verbose=1,
)
X, y = shuffle(X, y, random_state=123)
net.fit(X, y)
_, X_valid, _, y_valid = net.train_test_split(X, y, net.eval_size)
probas = net.predict_proba(X_valid)[:, 0]
print("ROC score", metrics.roc_auc_score(y_valid, probas))
return net, mean, std
net8 = NeuralNet(layers=[('input', layers.InputLayer),
('conv1', layers.Conv2DLayer),
('pool1', layers.MaxPool2DLayer),
('dropout1', layers.DropoutLayer),
('conv2', layers.Conv2DLayer),
('pool2', layers.MaxPool2DLayer),
('dropout2', layers.DropoutLayer),
('conv3', layers.Conv2DLayer),
('pool3', layers.MaxPool2DLayer),
('dropout3', layers.DropoutLayer),
('hidden4', layers.DenseLayer),
('dropout4', layers.DropoutLayer),
('hidden5', layers.DenseLayer),
('output', layers.DenseLayer)],
input_shape=(None, 1, 96, 96),
conv1_num_filters=32,
conv1_filter_size=(3, 3),
pool1_pool_size=(2, 2),
dropout1_p=0.1,
conv2_num_filters=64,
conv2_filter_size=(2, 2),
pool2_pool_size=(2, 2),
dropout2_p=0.2,
conv3_num_filters=128,
conv3_filter_size=(2, 2),
pool3_pool_size=(2, 2),
dropout3_p=0.3,
hidden4_num_units=1000,
dropout4_p=0.5,
hidden5_num_units=1000,
output_num_units=30,
output_nonlinearity=None,
update_learning_rate=theano.shared(float32(0.03)),
update_momentum=theano.shared(float32(0.9)),
regression=True,
batch_iterator_train=FlipBatchIterator(batch_size=128),
on_epoch_finished=[
AdjustVariable('update_learning_rate',
start=0.03,
stop=0.0001),
AdjustVariable('update_momentum', start=0.9, stop=0.999),
EarlyStopping(patience=200)
],
max_epochs=10000,
verbose=1)
from nolearn.lasagne import NeuralNet
from nnet.prepare import normalize_data
from settings import NCLASSES, VERBOSITY
from utils.shuffling import shuffle
X, y, features = normalize_data()
net = NeuralNet(
layers=[
('input', layers.InputLayer),
('hidden', layers.DenseLayer),
('output', layers.DenseLayer),
],
input_shape=(128, 93),
hidden_num_units=25,
output_nonlinearity=softmax,
output_num_units=NCLASSES,
update=nesterov_momentum,
update_learning_rate=0.0001,
update_momentum=0.9,
regression=False,
max_epochs=500,
verbose=bool(VERBOSITY),
)
B, j, key = shuffle(data=X, classes=y)
j -= 1
print B.shape
out = net.fit(B, j)
prediction = net.predict(B)
net2_aug = NeuralNet(
layers=[
('input', layers.InputLayer),
('conv1', layers.Conv2DLayer),
('pool1', layers.MaxPool2DLayer),
('conv2', layers.Conv2DLayer),
('pool2', layers.MaxPool2DLayer),
('conv3', layers.Conv2DLayer),
('pool3', layers.MaxPool2DLayer),
('hidden4', layers.DenseLayer),
('hidden5', layers.DenseLayer),
('output', layers.DenseLayer),
],
input_shape=(None, 1, 96, 96),
conv1_num_filters=32,
conv1_filter_size=(3, 3),
pool1_pool_size=(2, 2),
conv2_num_filters=64,
conv2_filter_size=(2, 2),
pool2_pool_size=(2, 2),
conv3_num_filters=128,
conv3_filter_size=(2, 2),
pool3_pool_size=(2, 2),
hidden4_num_units=500,
hidden5_num_units=500,
output_num_units=30,
output_nonlinearity=None,
update_learning_rate=0.01,
update_momentum=0.9,
regression=True,
batch_iterator_train=FlipBatchIterator(batch_size=128),
max_epochs=1000,
verbose=1,
)
('output', DenseLayer),
]
ae = NeuralNet(
layers=layers,
max_epochs=50,
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.975,
input_shape=(None, num_features),
dense_num_units=64,
narrow_num_units=48,
denseReverse1_num_units=64,
denseReverse2_num_units=128,
output_num_units=128,
#input_nonlinearity = None, #nonlinearities.sigmoid,
#dense_nonlinearity = nonlinearities.tanh,
narrow_nonlinearity=nonlinearities.softplus,
#denseReverse1_nonlinearity = nonlinearities.tanh,
denseReverse2_nonlinearity=nonlinearities.softplus,
output_nonlinearity=nonlinearities.linear, #nonlinearities.softmax,
#dropout0_p=0.1,
dropout1_p=0.01,
dropout2_p=0.001,
regression=True,
verbose=1)
ae.initialize()
PrintLayerInfo()(ae)
def classif(self, NeuralNet, X, y):
l = InputLayer(shape=(None, X.shape[1]))
l = DenseLayer(l, num_units=len(np.unique(y)), nonlinearity=softmax)
net = NeuralNet(l, update_learning_rate=0.01)
return net.fit(X, y)
def NN():
net = NeuralNet(
layers=[
('input', layers.InputLayer),
('conv1', layers.Conv2DLayer), # 2C 1MP
('conv2', layers.Conv2DLayer),
('pool1', layers.MaxPool2DLayer),
('cnndrop1', layers.DropoutLayer),
('conv3', layers.Conv2DLayer),
('conv4', layers.Conv2DLayer), # 2C 1 MP
('pool2', layers.MaxPool2DLayer),
('cnndrop2', layers.DropoutLayer),
('conv5', layers.Conv2DLayer),
('conv6', layers.Conv2DLayer), # 2C 1MP
('conv7', layers.Conv2DLayer),
('pool3', layers.MaxPool2DLayer),
('cnndrop3', layers.DropoutLayer),
('conv8', layers.Conv2DLayer),
('conv9', layers.Conv2DLayer), # 3C 1MP
('conv10', layers.Conv2DLayer),
('pool4', layers.MaxPool2DLayer),
('cnndrop4', layers.DropoutLayer),
('conv11', layers.Conv2DLayer),
('conv12', layers.Conv2DLayer),
('pool5', layers.MaxPool2DLayer), # 2C 1MP
('dropout1', layers.DropoutLayer),
('hidden1', layers.DenseLayer),
('maxout1', layers.pool.FeaturePoolLayer),
('dropout2', layers.DropoutLayer),
('hidden2', layers.DenseLayer),
('maxout2', layers.pool.FeaturePoolLayer),
('output', layers.DenseLayer)
],
input_shape=(None, 1, 512, 512),
conv1_num_filters=16,
conv1_filter_size=(5, 5),
conv1_stride=(2, 2),
conv1_nonlinearity=lasagne.nonlinearities.LeakyRectify(0.01),
conv1_pad=2,
conv2_num_filters=16,
conv2_filter_size=(3, 3),
conv2_stride=(1, 1),
conv2_nonlinearity=lasagne.nonlinearities.LeakyRectify(0.01),
conv2_pad=1,
pool1_pool_size=(2, 2),
cnndrop1_p=0.1,
conv3_num_filters=32,
conv3_filter_size=(3, 3),
conv3_stride=(2, 2),
conv3_nonlinearity=lasagne.nonlinearities.LeakyRectify(0.01),
conv3_pad=1,
conv4_num_filters=32,
conv4_filter_size=(3, 3),
conv4_stride=(1, 1),
conv4_nonlinearity=lasagne.nonlinearities.LeakyRectify(0.01),
conv4_pad=1,
pool2_pool_size=(2, 2),
cnndrop2_p=0.2,
conv5_num_filters=48,
conv5_filter_size=(3, 3),
conv5_stride=(1, 1),
conv5_nonlinearity=lasagne.nonlinearities.LeakyRectify(0.01),
conv5_pad=1,
conv6_num_filters=48,
conv6_filter_size=(3, 3),
conv6_stride=(1, 1),
conv6_nonlinearity=lasagne.nonlinearities.LeakyRectify(0.01),
conv6_pad=1,
conv7_num_filters=48,
conv7_filter_size=(3, 3),
conv7_stride=(1, 1),
conv7_nonlinearity=lasagne.nonlinearities.LeakyRectify(0.01),
conv7_pad=1,
pool3_pool_size=(2, 2),
cnndrop3_p=0.3,
conv8_num_filters=64,
conv8_filter_size=(3, 3),
conv8_stride=(1, 1),
conv8_nonlinearity=lasagne.nonlinearities.LeakyRectify(0.01),
conv8_pad=1,
conv9_num_filters=64,
conv9_filter_size=(3, 3),
conv9_stride=(1, 1),
conv9_nonlinearity=lasagne.nonlinearities.LeakyRectify(0.01),
conv9_pad=1,
conv10_num_filters=64,
conv10_filter_size=(3, 3),
conv10_stride=(1, 1),
conv10_nonlinearity=lasagne.nonlinearities.LeakyRectify(0.01),
conv10_pad=1,
pool4_pool_size=(2, 2),
cnndrop4_p=0.4,
conv11_num_filters=128,
conv11_filter_size=(3, 3),
conv11_stride=(1, 1),
conv11_nonlinearity=lasagne.nonlinearities.LeakyRectify(0.01),
conv11_pad=1,
conv12_num_filters=128,
conv12_filter_size=(3, 3),
conv12_stride=(1, 1),
conv12_nonlinearity=lasagne.nonlinearities.LeakyRectify(0.01),
conv12_pad=1,
pool5_pool_size=(2, 2),
dropout1_p=0.5,
hidden1_num_units=128,
hidden1_nonlinearity=lasagne.nonlinearities.LeakyRectify(0.01),
maxout1_pool_size=2,
dropout2_p=0.5,
hidden2_num_units=128,
hidden2_nonlinearity=lasagne.nonlinearities.LeakyRectify(0.01),
maxout2_pool_size=2,
#regression = True,
# output_nonlinearity=None,
output_num_units=5,
output_nonlinearity=lasagne.nonlinearities.softmax,
update=nesterov_momentum,
update_learning_rate=theano.shared(np.cast['float32'](0.003)),
#update_learning_rate = 0.003,
update_momentum=0.9,
on_epoch_finished=[updateRate.AdjustVariable('update_learning_rate')],
# objective_l2 = 0.0005,
max_epochs=300,
verbose=1,
)
return net
def regr(self, NeuralNet, X, y):
l = InputLayer(shape=(None, X.shape[1]))
l = DenseLayer(l, num_units=y.shape[1], nonlinearity=None)
net = NeuralNet(l, regression=True, update_learning_rate=0.01)
return net.fit(X, y)
net1 = NeuralNet(
# defining layers, three layers
layers=[
('input' , layers.InputLayer),
('hidden', layers.DenseLayer),
('output', layers.DenseLayer),
],
input_shape = (None, 9216), # 96*96 input pixels per batch
hidden_num_units = 100, # number of units in hidden layer
output_nonlinearity = None, # output layer uses identity function
output_num_units = 30, # 30 target values
# optimization method
#update = nesterov_momentum,
update = adagrad,
update_learning_rate = 0.01,
#update_momentum = 0.9,
regression = True, # a flag indicating we're dealing with regression problem
max_epochs = 50, # we want to train this many epochs
verbose=1,
)
Xb[i, c][mask] = 0.0
if yb != None:
yb = yb.astype(np.uint8)
return Xb, yb
test_iterator = RadialBatchIterator(batch_size=1)
net = phf.build_GoogLeNet(PATCH_SIZE, PATCH_SIZE)
nn = NeuralNet(
net['softmax'],
max_epochs=1,
update=adam,
update_learning_rate=.00014, #start with a really low learning rate
#objective_l2=0.0001,
# batch iteration params
batch_iterator_test=test_iterator,
train_split=TrainSplit(eval_size=0.2),
verbose=3,
)
nn.load_params_from(netfile)
nuclei_area = 0.0
mitosis_area = 0.0
num = 0
features = []
for img in images:
def load_dbn(path='models/oulu_ae.mat'):
"""
load a pretrained dbn from path
:param path: path to the .mat dbn
:return: pretrained deep belief network
"""
# create the network using weights from pretrain_nn.mat
nn = sio.loadmat(path)
w1 = nn['w1']
w2 = nn['w2']
w3 = nn['w3']
w4 = nn['w4']
w5 = nn['w5']
w6 = nn['w6']
w7 = nn['w7']
w8 = nn['w8']
b1 = nn['b1'][0]
b2 = nn['b2'][0]
b3 = nn['b3'][0]
b4 = nn['b4'][0]
b5 = nn['b5'][0]
b6 = nn['b6'][0]
b7 = nn['b7'][0]
b8 = nn['b8'][0]
layers = [
(InputLayer, {
'name': 'input',
'shape': (None, 1144)
}),
(DenseLayer, {
'name': 'l1',
'num_units': 2000,
'nonlinearity': sigmoid,
'W': w1,
'b': b1
}),
(DenseLayer, {
'name': 'l2',
'num_units': 1000,
'nonlinearity': sigmoid,
'W': w2,
'b': b2
}),
(DenseLayer, {
'name': 'l3',
'num_units': 500,
'nonlinearity': sigmoid,
'W': w3,
'b': b3
}),
(DenseLayer, {
'name': 'l4',
'num_units': 50,
'nonlinearity': linear,
'W': w4,
'b': b4
}),
(DenseLayer, {
'name': 'l5',
'num_units': 500,
'nonlinearity': sigmoid,
'W': w5,
'b': b5
}),
(DenseLayer, {
'name': 'l6',
'num_units': 1000,
'nonlinearity': sigmoid,
'W': w6,
'b': b6
}),
(DenseLayer, {
'name': 'l7',
'num_units': 2000,
'nonlinearity': sigmoid,
'W': w7,
'b': b7
}),
(DenseLayer, {
'name': 'output',
'num_units': 1144,
'nonlinearity': linear,
'W': w8,
'b': b8
}),
]
dbn = NeuralNet(
layers=layers,
max_epochs=30,
objective_loss_function=squared_error,
update=nesterov_momentum,
regression=True,
verbose=1,
update_learning_rate=0.001,
update_momentum=0.05,
objective_l2=0.005,
)
return dbn
(Conv2DLayer, {'num_filters': 128, 'filter_size': 3}),
(MaxPool2DLayer, {'pool_size': 2}),
(DenseLayer, {'num_units': 64}),
(DropoutLayer, {}),
(DenseLayer, {'num_units': 64}),
(DenseLayer, {'num_units': 10, 'nonlinearity': softmax}),
]
# net
net0 = NeuralNet(
layers = layers0,
max_epochs = 10,
update = adam, # For 'adam', a small learning rate is best
update_learning_rate = 0.0002,
objective_l2 = 0.0025, # L2 regularization
train_split = TrainSplit(eval_size = 0.25),
verbose = 1
)
net0.fit(X_train, y_train)
# visualization
from nolearn.lasagne.visualize import draw_to_notebook, plot_loss
from nolearn.lasagne.visualize import plot_conv_weights, plot_conv_activity
from nolearn.lasagne.visualize import plot_occlusion, plot_saliency
draw_to_notebook(net0)
plot_loss(net0)
#plot helps determine if we are overfitting:
net = NeuralNet(
layers=[
('input', InputLayer),
('hidden1', DenseLayer),
('dropout1', DropoutLayer),
('hidden2', DenseLayer),
('dropout2', DropoutLayer),
('output', DenseLayer),
],
# layer parameters:
input_shape=(None, input_size),
hidden1_num_units=850,
dropout1_p=0.1,
hidden2_num_units=200,
dropout2_p=0.10,
output_nonlinearity=sigmoid,
output_num_units=1,
# optimization method:
update=nesterov_momentum,
update_learning_rate=learning_rate,
update_momentum=0.91,
# Decay the learning rate
on_epoch_finished=[
AdjustVariable(learning_rate, target=0, half_life=4),
],
# This is silly, but we don't want a stratified K-Fold here
# To compensate we need to pass in the y_tensor_type and the loss.
regression=True,
y_tensor_type=T.imatrix,
objective_loss_function=binary_crossentropy,
max_epochs=50,
eval_size=0.2,
verbose=1,
)
net = NeuralNet(
layers=[
('input', InputLayer),
('dropout0', DropoutLayer),
('dense1', DenseLayer),
('dropout1', DropoutLayer),
('dense2', DenseLayer),
('dropout2', DropoutLayer),
('dense3', DenseLayer),
('dropout3', DropoutLayer),
('output', DenseLayer),
],
update=nesterov_momentum, #Todo: optimize
loss=None,
objective=Objective,
regression=False,
max_epochs=1000,
eval_size=0.1,
#on_epoch_finished = None,
#on_training_finished = None,
verbose=bool(VERBOSITY),
input_shape=(None, train.shape[1]),
output_num_units=NCLASSES,
dense1_num_units=500,
dense2_num_units=500,
dense3_num_units=400,
dense1_nonlinearity=LeakyRectify(leakiness=0.1),
dense2_nonlinearity=LeakyRectify(leakiness=0.1),
dense3_nonlinearity=LeakyRectify(leakiness=0.1),
output_nonlinearity=softmax,
dense1_W=HeUniform(),
dense2_W=HeUniform(),
dense3_W=HeUniform(),
dense1_b=Constant(0.),
dense2_b=Constant(0.),
dense3_b=Constant(0.),
output_b=Constant(0.),
dropout0_p=0.1,
dropout1_p=0.6,
dropout2_p=0.6,
dropout3_p=0.6,
update_learning_rate=shared(float32(0.02)), #
update_momentum=shared(float32(0.9)), #
batch_iterator_train=BatchIterator(batch_size=128),
batch_iterator_test=BatchIterator(batch_size=128),
)
net2 = NeuralNet(
layers=[
('input', layers.InputLayer),
('conv1', Conv2DLayer),
('pool1', MaxPool2DLayer),
('conv2', Conv2DLayer),
('conv3', Conv2DLayer),
('pool3', MaxPool2DLayer),
('hidden1', layers.DenseLayer),
('dropout1', layers.DropoutLayer),
('hidden2', layers.DenseLayer),
('dropout2', layers.DropoutLayer),
('hidden3', layers.DenseLayer),
('dropout3', layers.DropoutLayer),
('output', layers.DenseLayer),
],
input_shape=(None, 1, IMG_SIZE, IMG_SIZE),
conv1_num_filters=128,
conv1_filter_size=(5, 5),
conv1_pad=2,
conv1_strides=(4,4),
conv1_nonlinearity=lasagne.nonlinearities.rectify,
pool1_ds=(3, 3),
pool1_strides=(2,2),
conv2_num_filters=128,
conv2_filter_size=(3, 3),
conv2_pad=2,
conv2_nonlinearity=lasagne.nonlinearities.rectify,
conv3_num_filters=256,
conv3_filter_size=(3, 3),
conv3_pad=1,
conv3_nonlinearity=lasagne.nonlinearities.rectify,
pool3_ds=(3, 3),
pool3_strides=(2,2),
hidden1_num_units=512,
hidden1_nonlinearity=lasagne.nonlinearities.rectify,
dropout1_p=0.3,
hidden2_num_units=1024,
hidden2_nonlinearity=lasagne.nonlinearities.rectify,
dropout2_p=0.5,
hidden3_num_units=1024,
hidden3_nonlinearity=lasagne.nonlinearities.rectify,
dropout3_p=0.5,
output_num_units=121,
output_nonlinearity=lasagne.nonlinearities.softmax,
update_learning_rate=theano.shared(float32(0.01)),
update_momentum=theano.shared(float32(0.9)),
regression=False,
#loss=lasagne.objectives.multinomial_nll,
use_label_encoder=True,
batch_iterator_train=FlipBatchIterator(batch_size=256),
on_epoch_finished=[
AdjustVariable('update_learning_rate', start=0.01, stop=0.0001),
AdjustVariable('update_momentum', start=0.9, stop=0.999),
EarlyStopping(patience=50),
],
max_epochs=500,
verbose=2,
test_size=0.1
)
net = NeuralNet(eval_size=0.05,
layers=[
('input', layers.InputLayer),
('l1c', Conv2DCCLayer),
('l1p', MaxPool2DCCLayer),
('l1d', layers.DropoutLayer),
('l2c', Conv2DCCLayer),
('l2p', MaxPool2DCCLayer),
('l2d', layers.DropoutLayer),
('l3c', Conv2DCCLayer),
('l3d', layers.DropoutLayer),
('l4c', Conv2DCCLayer),
('l4d', layers.DropoutLayer),
('l5f', layers.DenseLayer),
('l5p', layers.FeaturePoolLayer),
('l5d', layers.DropoutLayer),
('l6f', layers.DenseLayer),
('l6p', layers.FeaturePoolLayer),
('l6d', layers.DropoutLayer),
('l7f', layers.DenseLayer),
('l7p', layers.FeaturePoolLayer),
('l7d', layers.DropoutLayer),
('output', layers.DenseLayer),
],
input_shape=(None, 1, 48, 48),
l1c_num_filters=64,
l1c_filter_size=(3, 3),
l1p_ds=(2, 2),
l1d_p=0.2,
l2c_num_filters=128,
l2c_filter_size=(3, 3),
l2p_ds=(2, 2),
l2d_p=0.3,
l3c_num_filters=256,
l3c_filter_size=(3, 3),
l3d_p=0.4,
l4c_num_filters=512,
l4c_filter_size=(3, 3),
l4d_p=0.4,
l5f_num_units=2048,
l5p_ds=2,
l5d_p=0.5,
l6f_num_units=2048,
l6p_ds=2,
l6d_p=0.5,
l7f_num_units=2048,
l7p_ds=2,
l7d_p=0.5,
output_num_units=121,
output_nonlinearity=softmax,
loss=multinomial_nll,
batch_iterator_train=train_iterator,
batch_iterator_test=test_iterator,
update=rmsprop,
update_learning_rate=theano.shared(float32(5e-5)),
update_rho=0.9,
update_epsilon=1e-6,
regression=False,
on_epoch_finished=[
StepDecay('update_learning_rate', start=5e-5, stop=1e-7),
EarlyStopping(patience=100)
],
max_epochs=1500,
verbose=1)
net = NeuralNet(
eval_size=0.05,
layers=[
('input', layers.InputLayer),
('l1c', Conv2DCCLayer),
('l1p', MaxPool2DCCLayer),
('l1d', layers.DropoutLayer),
('l2c', Conv2DCCLayer),
('l2p', MaxPool2DCCLayer),
('l2d', layers.DropoutLayer),
('l3c', Conv2DCCLayer),
('l3d', layers.DropoutLayer),
('l5f', layers.DenseLayer),
('l5d', layers.DropoutLayer),
('l6f', layers.DenseLayer),
('l6d', layers.DropoutLayer),
('l7f', layers.DenseLayer),
('l7d', layers.DropoutLayer),
('output', layers.DenseLayer),
],
input_shape=(None, 1, 48, 48),
l1c_num_filters=64,
l1c_filter_size=(5, 5),
l1p_ds=(2, 2),
l1d_p=0.2,
l2c_num_filters=128,
l2c_filter_size=(5, 5),
l2p_ds=(2, 2),
l2d_p=0.2,
l3c_num_filters=256,
l3c_filter_size=(3, 3),
l3d_p=0.2,
l5f_num_units=1024,
l5d_p=0.5,
l6f_num_units=1024,
l6d_p=0.5,
l7f_num_units=1024,
l7d_p=0.5,
output_num_units=121,
output_nonlinearity=softmax,
loss=multinomial_nll,
batch_iterator_train=BatchIterator(batch_size=128),
batch_iterator_test=BatchIterator(batch_size=128),
update=adadelta,
update_learning_rate=theano.shared(float32(1e-0)),
update_rho=0.95,
update_epsilon=1e-6,
regression=False,
on_epoch_finished=[
# StepDecay('update_learning_rate', start=1e-5, stop=1e-7),
StepDecay('update_learning_rate', start=1e-0, stop=1e-2),
EarlyStopping(patience=100)
],
max_epochs=1000,
verbose=1)
def build_nn(x_train, y_train, x_test, y_test):
"""
Construct a regression neural network model from input dataframe
:param x_train: features dataframe for model training
:param y_train: target dataframe for model training
:param x_test: features dataframe for model testing
:param y_test: target dataframe for model testing
:return: None
"""
# Create classification model
net = NeuralNet(layers=[('input', InputLayer), ('hidden0', DenseLayer),
('hidden1', DenseLayer), ('output', DenseLayer)],
input_shape=(None, x_train.shape[1]),
hidden0_num_units=NODES,
hidden0_nonlinearity=nonlinearities.softmax,
hidden1_num_units=NODES,
hidden1_nonlinearity=nonlinearities.softmax,
output_num_units=len(np.unique(y_train)),
output_nonlinearity=nonlinearities.softmax,
update_learning_rate=0.1,
verbose=1,
max_epochs=100)
param_grid = {
'hidden0_num_units': [1, 4, 17, 25],
'hidden0_nonlinearity':
[nonlinearities.sigmoid, nonlinearities.softmax],
'hidden1_num_units': [1, 4, 17, 25],
'hidden1_nonlinearity':
[nonlinearities.sigmoid, nonlinearities.softmax],
'update_learning_rate': [0.01, 0.1, 0.5]
}
grid = sklearn.grid_search.GridSearchCV(net,
param_grid,
verbose=0,
n_jobs=3,
cv=3)
grid.fit(x_train, y_train)
y_pred = grid.predict(x_test)
# Mean absolute error regression loss
mean_abs = sklearn.metrics.mean_absolute_error(y_test, y_pred)
# Mean squared error regression loss
mean_sq = sklearn.metrics.mean_squared_error(y_test, y_pred)
# Median absolute error regression loss
median_abs = sklearn.metrics.median_absolute_error(y_test, y_pred)
# R^2 (coefficient of determination) regression score function
r2 = sklearn.metrics.r2_score(y_test, y_pred)
# Explained variance regression score function
exp_var_score = sklearn.metrics.explained_variance_score(y_test, y_pred)
# Accuracy prediction score
accuracy = sklearn.metrics.accuracy_score(y_test, y_pred)
with open(NN_PICKLE, 'wb') as results:
pickle.dump(grid, results, pickle.HIGHEST_PROTOCOL)
pickle.dump(net, results, pickle.HIGHEST_PROTOCOL)
pickle.dump(mean_abs, results, pickle.HIGHEST_PROTOCOL)
pickle.dump(mean_sq, results, pickle.HIGHEST_PROTOCOL)
pickle.dump(median_abs, results, pickle.HIGHEST_PROTOCOL)
pickle.dump(r2, results, pickle.HIGHEST_PROTOCOL)
pickle.dump(exp_var_score, results, pickle.HIGHEST_PROTOCOL)
pickle.dump(accuracy, results, pickle.HIGHEST_PROTOCOL)
pickle.dump(y_pred, results, pickle.HIGHEST_PROTOCOL)
return
def define_net():
define_net_specific_parameters()
io = ImageIO()
# Read pandas csv labels
y = util.load_labels()
if params.SUBSET is not 0:
y = y[:params.SUBSET]
X = np.arange(y.shape[0])
mean, std = io.load_mean_std(circularized=params.CIRCULARIZED_MEAN_STD)
keys = y.index.values
if params.AUGMENT:
train_iterator = AugmentingParallelBatchIterator(keys,
params.BATCH_SIZE,
std,
mean,
y_all=y)
else:
train_iterator = ParallelBatchIterator(keys,
params.BATCH_SIZE,
std,
mean,
y_all=y)
test_iterator = ParallelBatchIterator(keys,
params.BATCH_SIZE,
std,
mean,
y_all=y)
if params.REGRESSION:
y = util.float32(y)
y = y[:, np.newaxis]
if 'gpu' in theano.config.device:
# Half of coma does not support cuDNN, check whether we can use it on this node
# If not, use cuda_convnet bindings
from theano.sandbox.cuda.dnn import dnn_available
if dnn_available() and not params.DISABLE_CUDNN:
from lasagne.layers import dnn
Conv2DLayer = dnn.Conv2DDNNLayer
MaxPool2DLayer = dnn.MaxPool2DDNNLayer
else:
from lasagne.layers import cuda_convnet
Conv2DLayer = cuda_convnet.Conv2DCCLayer
MaxPool2DLayer = cuda_convnet.MaxPool2DCCLayer
else:
Conv2DLayer = layers.Conv2DLayer
MaxPool2DLayer = layers.MaxPool2DLayer
Maxout = layers.pool.FeaturePoolLayer
net = NeuralNet(
layers=[
('input', layers.InputLayer),
('conv0', Conv2DLayer),
('pool0', MaxPool2DLayer),
('conv1', Conv2DLayer),
('pool1', MaxPool2DLayer),
('conv2', Conv2DLayer),
('pool2', MaxPool2DLayer),
('conv3', Conv2DLayer),
('pool3', MaxPool2DLayer),
('conv4', Conv2DLayer),
('pool4', MaxPool2DLayer),
('dropouthidden1', layers.DropoutLayer),
('hidden1', layers.DenseLayer),
('maxout1', Maxout),
('dropouthidden2', layers.DropoutLayer),
('hidden2', layers.DenseLayer),
('maxout2', Maxout),
('dropouthidden3', layers.DropoutLayer),
('output', layers.DenseLayer),
],
input_shape=(None, params.CHANNELS, params.PIXELS, params.PIXELS),
conv0_num_filters=32,
conv0_filter_size=(5, 5),
conv0_stride=(2, 2),
pool0_pool_size=(2, 2),
pool0_stride=(2, 2),
conv1_num_filters=64,
conv1_filter_size=(3, 3),
conv1_border_mode='same',
pool1_pool_size=(2, 2),
pool1_stride=(2, 2),
conv2_num_filters=128,
conv2_filter_size=(3, 3),
conv2_border_mode='same',
pool2_pool_size=(2, 2),
pool2_stride=(2, 2),
conv3_num_filters=192,
conv3_filter_size=(3, 3),
conv3_border_mode='same',
pool3_pool_size=(2, 2),
pool3_stride=(2, 2),
conv4_num_filters=256,
conv4_filter_size=(3, 3),
conv4_border_mode='same',
pool4_pool_size=(2, 2),
pool4_stride=(2, 2),
hidden1_num_units=1024,
hidden2_num_units=1024,
dropouthidden1_p=0.5,
dropouthidden2_p=0.5,
dropouthidden3_p=0.5,
maxout1_pool_size=2,
maxout2_pool_size=2,
output_num_units=1 if params.REGRESSION else 5,
output_nonlinearity=None
if params.REGRESSION else nonlinearities.softmax,
update_learning_rate=theano.shared(
util.float32(params.START_LEARNING_RATE)),
update_momentum=theano.shared(util.float32(params.MOMENTUM)),
custom_score=('kappa', quadratic_kappa),
regression=params.REGRESSION,
batch_iterator_train=train_iterator,
batch_iterator_test=test_iterator,
on_epoch_finished=[
AdjustVariable('update_learning_rate',
start=params.START_LEARNING_RATE),
stats.Stat(),
ModelSaver()
],
max_epochs=500,
verbose=1,
# Only relevant when create_validation_split = True
eval_size=0.1,
# Need to specify splits manually like indicated below!
create_validation_split=params.SUBSET > 0,
)
# It is recommended to use the same training/validation split every model for ensembling and threshold optimization
#
# To set specific training/validation split:
net.X_train = np.load(params.IMAGE_SOURCE + "/X_train.npy")
net.X_valid = np.load(params.IMAGE_SOURCE + "/X_valid.npy")
net.y_train = np.load(params.IMAGE_SOURCE + "/y_train.npy")
net.y_valid = np.load(params.IMAGE_SOURCE + "/y_valid.npy")
return net, X, y
num_classes = len(encoder.classes_)
num_features = X.shape[1]
net1 = NeuralNet(
layers=[ # three layers: one hidden layer
('input', layers.InputLayer),
('hidden', layers.DenseLayer),
('dropout', layers.DropoutLayer),
('output', layers.DenseLayer),
],
# layer parameters:
input_shape=(None, num_features),
hidden_num_units=200, # number of units in hidden layer #!200-600
output_nonlinearity=lasagne.nonlinearities.softmax, # output layer
output_num_units=num_classes, # 10 target values
dropout_p=0.2,
#!dropout 0.2-0.7
# optimization method:
update=nesterov_momentum,
update_learning_rate=0.01, #!0.001-0.01
update_momentum=0.9, #!0.6-0.9
regression=
False, # flag to indicate we're dealing with regression problem
max_epochs=500, # we want to train this many epochs
verbose=1,
)
random_search = RandomizedSearchCV(
net1, {
'hidden_num_units': sp_randint(200, 600),