def gridsearch_alpha(self,learning_rate,index,params=None):
hidden_unit = ((index+1)*2)/3
self.l_in = ls.layers.InputLayer(shape=(None,n_input),input_var=None)
self.l_hidden = ls.layers.DenseLayer(self.l_in,num_units=15,nonlinearity=ls.nonlinearities.rectify)
self.network = l_out = ls.layers.DenseLayer(self.l_hidden,num_units=1)
list_results = np.array([learning_rate.shape[0]],dtype=np.float64)
for item in learning_rate:
#Init Neural net
net1 = NeuralNet(
layers=self.network,
# optimization method:
update=nesterov_momentum,
update_learning_rate=item,
update_momentum=0.9,
regression=True, # flag to indicate we're dealing with regression problem
max_epochs=800, # we want to train this many epochs
# verbose=1,
eval_size = 0.4
)
net1.fit(self.X_training,self.y_training)
self.pred = net1.predict(self.n_sample2)
name_file = "Params/saveNeuralNetwork_%s_%s.tdn" %(item,index)
net1.save_params_to(name_file)
score_nn = net1.score(self.n_sample2,self.n_test2)
list_results[item] = score_nn
print "index=%f,item=%f,score=%f"%(index,item,score_nn)
return list_results
def gridsearch_alpha(self, learning_rate, index, params=None):
self.l_in = ls.layers.InputLayer(shape=(None, n_input),
input_var=None,
W=params.T)
self.l_hidden = ls.layers.DenseLayer(
self.l_in, num_units=15, nonlinearity=ls.nonlinearities.rectify)
self.network = l_out = ls.layers.DenseLayer(self.l_hidden, num_units=1)
list_results = np.array([learning_rate.shape[0]], dtype=np.float64)
for item in learning_rate:
#Init Neural net
net1 = NeuralNet(
layers=self.network,
# optimization method:
update=nesterov_momentum,
update_learning_rate=item,
update_momentum=0.9,
regression=
True, # flag to indicate we're dealing with regression problem
max_epochs=800, # we want to train this many epochs
# verbose=1,
eval_size=0.4)
#
net1.fit(self.X_training, self.y_training)
self.pred = net1.predict(self.n_sample2)
name_file = "GeneticParams/saveNeuralNetwork_%s_%s.tdn" % (item,
index)
net1.save_params_to(name_file)
score_nn = net1.score(self.n_sample2, self.n_test2)
list_results[item] = score_nn
print "index=%s,item=%f,score=%f" % (index, item, score_nn)
return list_results
def train_nn_model():
imageSize = 400 # 20 x 20 pixels
from lasagne import layers
from lasagne.updates import nesterov_momentum, sgd
from nolearn.lasagne import NeuralNet
model = NeuralNet(layers=[('input', layers.InputLayer),
('hidden', layers.DenseLayer),
('output', layers.DenseLayer),],
# layer parameters:
input_shape=(None, 400), # 96x96 input pixels per batch
hidden_num_units=100, # number of units in hidden layer
output_nonlinearity=None, # output layer uses identity function
output_num_units=62, # 30 target values
# optimization method:
update=nesterov_momentum,
update_learning_rate=0.01,
update_momentum=0.9,
regression=True, # flag to indicate we're dealing with regression problem
max_epochs=400, # we want to train this many epochs
verbose=1,)
xTrain, yTrain, xTest, labelsInfoTest = load_train_test_data(nn_ytrain=True)
print xTrain.shape, yTrain.shape
print xTrain.dtype, yTrain.dtype
model.fit(xTrain, yTrain)
ytest_pred = model.predict(xTrain)
print model.score(xTrain, yTrain)
print accuracy_score(ytest_pred, yTrain)
yTest = model.predict(xTest)
print labelsInfoTest.shape, yTest.shape
yTest2 = transform_to_class(yTest)
submit_df = labelsInfoTest
submit_df['Class'] = yTest2
submit_df.to_csv('submission.csv', index=False)
return model
def classify(X, y):
l = InputLayer(shape=(None, X.shape[1]))
l = DenseLayer(l, num_units=100, nonlinearity=tanh)
l = DenseLayer(l, num_units=len(np.unique(y)), nonlinearity=softmax)
net = NeuralNet(l,
update_learning_rate=0.05,
train_split=TrainSplit(eval_size=0.2),
verbose=1)
net.fit(X, y)
print(net.score(X, y))
def regr(X, Y): l = InputLayer(shape=(None, X.shape[1])) l = DenseLayer(l, num_units=X.shape[1], nonlinearity=tanh) #tanh, sigmoid # l = DropoutLayer(l, p=0.3, rescale=True) # previous: p=0.5 l = DenseLayer(l, num_units=1, nonlinearity=sigmoid) # l = DropoutLayer(l, p=0.3, rescale=True) # previous: p=0.5 net = NeuralNet(l, regression=True, update_learning_rate=0.01, verbose=1, max_epochs=700) net.fit(X, Y) print(net.score(X, Y)) return net
def regress(X, y):
l = InputLayer(shape=(None, X.shape[1]))
l = DenseLayer(l, num_units=100, nonlinearity=tanh)
l = DenseLayer(l, num_units=y.shape[1], nonlinearity=None)
net = NeuralNet(l,
regression=True,
update_learning_rate=0.05,
train_split=TrainSplit(eval_size=0.2),
verbose=1)
net.fit(X, y)
print(net.score(X, y))
def regr(X, Y): l = InputLayer(shape=(None, X.shape[1])) l = DenseLayer(l, num_units=Y.shape[1]+100, nonlinearity=tanh) # l = DropoutLayer(l, p=0.3, rescale=True) # previous: p=0.5 l = DenseLayer(l, num_units=Y.shape[1]+50, nonlinearity=tanh) # l = DropoutLayer(l, p=0.3, rescale=True) # previous: p=0.5 l = DenseLayer(l, num_units=Y.shape[1], nonlinearity=None) net = NeuralNet(l, regression=True, update_learning_rate=0.1, verbose=1) net.fit(X, Y) print(net.score(X, Y)) return net
def cancer(X, y, X_valid, y_valid):
l = InputLayer(shape=(None, X.shape[1]))
l = DenseLayer(l, num_units=len(np.unique(y)), nonlinearity=softmax)
net = NeuralNet(l, update_learning_rate=0.01, max_epochs=1000)
net.fit(X, y)
print(net.score(X, y))
y_pred = net.predict(X_valid)
print(y_valid)
print(y_pred)
plot_loss(net)
plt.title('Cancer')
plt.show()
def regr(X, y, X_valid, y_valid):
l = InputLayer(shape=(None, X.shape[1]))
l = DenseLayer(l, num_units=100, nonlinearity=softmax)
# l = DenseLayer(l, num_units=40, nonlinearity=softmax)
l = DenseLayer(l, num_units=len(np.unique(y)), nonlinearity=softmax)
net = NeuralNet(l,
update=adam,
update_learning_rate=0.01,
max_epochs=2000,
objective_loss_function=squared_error,
regression=True)
net.fit(X, y)
print(net.score(X, y))
y_pred = net.predict(X_valid)
print(y_valid)
print(y_pred)
plot_loss(net)
plt.title('Digits')
plt.show()
def test_lasagne_functional_regression(boston):
from nolearn.lasagne import NeuralNet
X, y = boston
layer1 = InputLayer(shape=(128, 13))
layer2 = DenseLayer(layer1, num_units=100)
output = DenseLayer(layer2, num_units=1, nonlinearity=identity)
nn = NeuralNet(
layers=output,
update_learning_rate=0.01,
update_momentum=0.1,
regression=True,
max_epochs=50,
)
nn.fit(X[:300], y[:300])
assert mean_absolute_error(nn.predict(X[300:]), y[300:]) < 3.0
assert r2_score(nn.predict(X[300:]), y[300:]) == nn.score(X[300:], y[300:])
def test_lasagne_functional_regression(boston):
from nolearn.lasagne import NeuralNet
X, y = boston
layer1 = InputLayer(shape=(128, 13))
layer2 = DenseLayer(layer1, num_units=100)
output = DenseLayer(layer2, num_units=1, nonlinearity=identity)
nn = NeuralNet(
layers=output,
update_learning_rate=0.01,
update_momentum=0.1,
regression=True,
max_epochs=50,
)
nn.fit(X[:300], y[:300])
assert mean_absolute_error(nn.predict(X[300:]), y[300:]) < 3.0
assert r2_score(nn.predict(X[300:]), y[300:]) == nn.score(X[300:], y[300:])
def classify(X, y, X_test, y_test):
layers0 = [('input', InputLayer),
('dense0', DenseLayer),
('dropout0', DropoutLayer),
('dense1', DenseLayer),
('dropout1', DropoutLayer),
('output', DenseLayer)]
net = NeuralNet(layers=layers0,
input_shape=(None, X.shape[1]),
dense0_num_units=300,
dropout0_p=0.075,
dropout1_p=0.1,
dense1_num_units=750,
output_num_units=3,
output_nonlinearity=softmax,
update=nesterov_momentum,
update_learning_rate=0.001,
update_momentum=0.99,
eval_size=0.2,
verbose=1,
max_epochs=15)
net.fit(X, y)
print(net.score(X, y))
preds = net.predict(X_test)
print(classification_report(y_test, preds))
cm = confusion_matrix(y_test, preds)
plt.matshow(cm)
plt.title('Confusion matrix')
plt.colorbar()
plt.ylabel('True label')
plt.xlabel('Predicted label')
plt.savefig('confmatrix.png')
plt.show()
print(cm)
def train_nolearn_model(X, y):
'''
NeuralNet with nolearn
'''
X = X.astype(np.float32)
y = y.astype(np.int32)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.2,
random_state=5)
X_train, X_test = impute_nan(X_train, X_test)
X_train, X_test = normalize_features(X_train, X_test)
lays = [
('input', layers.InputLayer),
('hidden', layers.DenseLayer),
('output', layers.DenseLayer),
]
net = NeuralNet(
layers=lays,
input_shape=(None, 23),
hidden_num_units=10,
objective_loss_function=lasagne.objectives.categorical_crossentropy,
output_nonlinearity=lasagne.nonlinearities.sigmoid,
output_num_units=10,
update=nesterov_momentum,
update_learning_rate=0.001,
update_momentum=0.9,
max_epochs=10,
verbose=1,
)
#net.fit(X_train, y_train)
#predicted = net.predict(X_test)
test_score = net.predict(X_test, y_test)
train_score = net.score(X_train, y_train)
return train_score, test_score
def train_nolearn_model(X, y):
'''
NeuralNet with nolearn
'''
X = X.astype(np.float32)
y = y.astype(np.int32)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 5)
X_train, X_test = impute_nan(X_train, X_test)
X_train, X_test = normalize_features(X_train, X_test)
lays = [('input', layers.InputLayer),
('hidden', layers.DenseLayer),
('output', layers.DenseLayer),
]
net = NeuralNet(
layers = lays,
input_shape=(None, 23),
hidden_num_units=10,
objective_loss_function=lasagne.objectives.categorical_crossentropy,
output_nonlinearity=lasagne.nonlinearities.sigmoid,
output_num_units=10,
update = nesterov_momentum,
update_learning_rate= 0.001,
update_momentum=0.9,
max_epochs=10,
verbose=1,
)
#net.fit(X_train, y_train)
#predicted = net.predict(X_test)
test_score = net.predict(X_test, y_test)
train_score = net.score(X_train, y_train)
return train_score, test_score
def build_nn(df=None, class_column_name=None):
"""
Construct a classification neural network model from input dataframe
Parameters:
df : input dataframe
class_column_name : identity of the column in df with class data
"""
# Type check inputs for sanity
if df is None:
raise ValueError('df is None')
if not isinstance(df, pd.DataFrame):
raise TypeError('df is not a dataframe')
if class_column_name is None:
raise ValueError('class_column_name is None')
if not isinstance(class_column_name, basestring):
raise TypeError('class_column_name is not a string')
if class_column_name not in df.columns:
raise ValueError('class_column_name (%s) is not a valid column name' %
class_column_name)
df = df.sample(frac=1).reset_index(drop=True)
df_train, df_test = train_test_split(df, TEST_SIZE)
df_train, df_val = df_train[:(0.75 * len(df_train.index)), :], df_train[(
0.75 * len(df_train.index)):, :]
x_train, x_val, x_test = df_train, df_val, df_test
# Remove the classification column from the dataframe
x_train = x_train.drop(class_column_name, axis=1, inplace=True).values
x_val = x_val.drop(class_column_name, axis=1, inplace=True).values
x_test = x_test.drop(class_column_name, axis=1, inplace=True).values
y_train = df_train[class_column_name].values.astype(np.int32)
y_val = df_val[class_column_name].values.astype(np.int32)
y_test = df_test[class_column_name].values.astype(np.int32)
# Create classification model
net = NeuralNet(layers=[('input', InputLayer), ('hidden0', DenseLayer),
('hidden1', DenseLayer), ('output', DenseLayer)],
input_shape=(None, x_train.shape[1]),
hidden0_num_units=NODES,
hidden0_nonlinearity=nonlinearities.softmax,
hidden1_num_units=NODES,
hidden1_nonlinearity=nonlinearities.softmax,
output_num_units=len(np.unique(y_train)),
output_nonlinearity=nonlinearities.softmax,
update_learning_rate=0.1,
verbose=1,
max_epochs=100)
param_grid = {
'hidden0_num_units': [4, 17, 25],
'hidden0_nonlinearity':
[nonlinearities.sigmoid, nonlinearities.softmax],
'hidden1_num_units': [4, 17, 25],
'hidden1_nonlinearity':
[nonlinearities.sigmoid, nonlinearities.softmax],
'update_learning_rate': [0.01, 0.1, 0.5]
}
grid_search = GridSearchCV(net, param_grid, verbose=0)
grid_search.fit(x_train, y_train)
net.fit(x_train, y_train)
print(net.score(x_train, y_train))
with open(PICKLE, 'wb') as file:
pickle.dump(x_train, file, pickle.HIGHEST_PROTOCOL)
pickle.dump(y_train, file, pickle.HIGHEST_PROTOCOL)
pickle.dump(df_test, file, pickle.HIGHEST_PROTOCOL)
pickle.dump(grid_search, file, pickle.HIGHEST_PROTOCOL)
pickle.dump(net, file, pickle.HIGHEST_PROTOCOL)
def classif(X, y):
l = InputLayer(shape=(None, X.shape[1]))
l = DenseLayer(l, num_units=len(np.unique(y)), nonlinearity=softmax)
net = NeuralNet(l, update_learning_rate=0.01)
net.fit(X, y)
print(net.score(X, y))
def regr(X, y):
l = InputLayer(shape=(None, X.shape[1]))
l = DenseLayer(l, num_units=y.shape[1], nonlinearity=None)
net = NeuralNet(l, regression=True, update_learning_rate=0.01)
net.fit(X, y)
print(net.score(X, y))
print 'started', datetime.now()
res = net0.fit(features_scaled, labels.astype('int32'))
print 'finished', datetime.now()
print res
# In[2]:
import pickle
pickle.dump(net0, open('lasagne_model.pkl', 'wb'))
# In[106]:
print 'score', net0.score(features_scaled, labels)
# In[111]:
print 'logloss', log_loss(labels, net0.predict_proba(features_scaled))
# In[16]:
test_features, _ = get_features(test)
scaler.fit(test_features)
test_features = scaler.transform(test_features)
class NNet(BaseEstimator, ClassifierMixin):
def __init__(
self,
name='nameless_net', # used for saving, so maybe make it unique
dense1_size=60,
dense1_nonlinearity='tanh',
dense1_init='orthogonal',
dense2_size=None,
dense2_nonlinearity=None, # inherits dense1
dense2_init=None, # inherits dense1
dense3_size=None,
dense3_nonlinearity=None, # inherits dense2
dense3_init=None, # inherits dense2
learning_rate=0.001,
learning_rate_scaling=100,
momentum=0.9,
momentum_scaling=100,
max_epochs=3000,
epoch_steps=None,
dropout0_rate=0, # this is the input layer
dropout1_rate=None,
dropout2_rate=None, # inherits dropout1_rate
dropout3_rate=None, # inherits dropout2_rate
weight_decay=0,
adaptive_weight_decay=False,
batch_size=128,
output_nonlinearity='softmax',
auto_stopping=True,
save_snapshots_stepsize=None,
):
"""
Create the network with the selected parameters.
:param name: Name for save files
:param dense1_size: Number of neurons for first hidden layer
:param dense1_nonlinearity: The activation function for the first hidden layer
:param dense1_init: The weight initialization for the first hidden layer
:param learning_rate: The (initial) learning rate (how fast the network learns)
:param learning_rate_scaling: The total factor to gradually decrease the learning rate by
:param momentum: The (initial) momentum
:param momentum_scaling: Similar to learning_rate_scaling
:param max_epochs: Total number of epochs (at most)
:param dropout1_rate: Percentage of connections dropped each step for first hidden layer
:param weight_decay: Palatalizes the weights by L2 norm (regularizes but decreases results)
:param adaptive_weight_decay: Should the weight decay adapt automatically?
:param batch_size: How many samples to send through the network at a time
:param auto_stopping: Stop early if the network seems to stop performing well
:param pretrain: Filepath of the previous weights to start at (or None)
:return:
"""
"""
Input argument storage: automatically store all locals, which should be exactly the arguments at this point, but storing a little too much is not a big problem.
"""
params = locals()
del params['self']
#self.__dict__.update(params)
self.parameter_names = sorted(params.keys())
"""
Check the parameters and update some defaults (will be done for 'self', no need to store again).
"""
self.set_params(**params)
def init_net(self,
feature_count,
class_count=NCLASSES,
verbosity=VERBOSITY >= 2):
"""
Initialize the network (needs to be done when data is available in order to set dimensions).
"""
if VERBOSITY >= 1:
print 'initializing network {0:s} {1:d}x{2:d}x{3:d}'.format(
self.name, self.dense1_size or 0, self.dense2_size or 0,
self.dense3_size or 0)
if VERBOSITY >= 2:
print 'parameters: ' + ', '.join(
'{0:s} = {1:}'.format(k, v)
for k, v in self.get_params(deep=False).items())
self.feature_count = feature_count
self.class_count = class_count
"""
Create the layers and their settings.
"""
self.layers = [
('input', InputLayer),
]
self.params = {
'dense1_num_units': self.dense1_size,
'dense1_nonlinearity': nonlinearities[self.dense1_nonlinearity],
'dense1_W': initializers[self.dense1_init],
'dense1_b': Constant(0.),
}
if self.dropout0_rate:
self.layers += [('dropout0', DropoutLayer)]
self.params['dropout0_p'] = self.dropout0_rate
self.layers += [
('dense1', DenseLayer),
]
if self.dropout1_rate:
self.layers += [('dropout1', DropoutLayer)]
self.params['dropout1_p'] = self.dropout1_rate
if self.dense2_size:
self.layers += [('dense2', DenseLayer)]
self.params.update({
'dense2_num_units':
self.dense2_size,
'dense2_nonlinearity':
nonlinearities[self.dense2_nonlinearity],
'dense2_W':
initializers[self.dense2_init],
'dense2_b':
Constant(0.),
})
else:
assert not self.dense3_size, 'There cannot be a third dense layer without a second one'
if self.dropout2_rate:
assert self.dense2_size is not None, 'There cannot be a second dropout layer without a second dense layer.'
self.layers += [('dropout2', DropoutLayer)]
self.params['dropout2_p'] = self.dropout2_rate
if self.dense3_size:
self.layers += [('dense3', DenseLayer)]
self.params.update({
'dense3_num_units':
self.dense3_size,
'dense3_nonlinearity':
nonlinearities[self.dense3_nonlinearity],
'dense3_W':
initializers[self.dense3_init],
'dense3_b':
Constant(0.),
})
if self.dropout3_rate:
assert self.dense2_size is not None, 'There cannot be a third dropout layer without a third dense layer.'
self.layers += [('dropout3', DropoutLayer)]
self.params['dropout3_p'] = self.dropout2_rate
self.layers += [('output', DenseLayer)]
self.params.update({
'output_nonlinearity':
nonlinearities[self.output_nonlinearity],
'output_W':
GlorotUniform(),
'output_b':
Constant(0.),
})
"""
Create meta parameters and special handlers.
"""
if VERBOSITY >= 3:
print 'learning rate: {0:.6f} -> {1:.6f}'.format(
abs(self.learning_rate),
abs(self.learning_rate) / float(self.learning_rate_scaling))
print 'momentum: {0:.6f} -> {1:.6f}'.format(
abs(self.momentum),
1 - ((1 - abs(self.momentum)) / float(self.momentum_scaling)))
self.step_handlers = [
LinearVariable('update_learning_rate',
start=abs(self.learning_rate),
stop=abs(self.learning_rate) /
float(self.learning_rate_scaling)),
LinearVariable(
'update_momentum',
start=abs(self.momentum),
stop=1 -
((1 - abs(self.momentum)) / float(self.momentum_scaling))),
StopNaN(),
]
self.end_handlers = [
SnapshotEndSaver(base_name=self.name),
TrainProgressPlotter(base_name=self.name),
]
snapshot_name = 'nn_' + params_name(self.params, prefix=self.name)[0]
if self.save_snapshots_stepsize:
self.step_handlers += [
SnapshotStepSaver(every=self.save_snapshots_stepsize,
base_name=snapshot_name),
]
if self.auto_stopping:
self.step_handlers += [
StopWhenOverfitting(loss_fraction=0.9,
base_name=snapshot_name),
StopAfterMinimum(patience=40, base_name=self.name),
]
weight_decay = shared(float32(abs(self.weight_decay)), 'weight_decay')
if self.adaptive_weight_decay:
self.step_handlers += [
AdaptiveWeightDecay(weight_decay),
]
if self.epoch_steps:
self.step_handlers += [
BreakEveryN(self.epoch_steps),
]
"""
Create the actual nolearn network with information from __init__.
"""
self.net = NeuralNet(
layers=self.layers,
objective=partial(WeightDecayObjective, weight_decay=weight_decay),
input_shape=(None, feature_count),
output_num_units=class_count,
update=nesterov_momentum, # todo: make parameter
update_learning_rate=shared(float32(self.learning_rate)),
update_momentum=shared(float(self.weight_decay)),
on_epoch_finished=self.step_handlers,
on_training_finished=self.end_handlers,
regression=False,
max_epochs=self.max_epochs,
verbose=verbosity,
batch_iterator_train=BatchIterator(batch_size=self.batch_size),
batch_iterator_test=BatchIterator(batch_size=self.batch_size),
eval_size=0.1,
#custom_score = ('custom_loss', categorical_crossentropy),
**self.params)
self.net.parent = self
self.net.initialize()
return self.net
def get_params(self, deep=True):
return OrderedDict(
(name, getattr(self, name)) for name in self.parameter_names)
def set_params(self, **params):
"""
Set all the parameters.
"""
for name, val in params.items():
assert name in self.parameter_names, '"{0:s}" is not a valid parameter name (known parameters: "{1:s}")'.format(
name, '", "'.join(self.parameter_names))
setattr(self, name, val)
"""
Arguments checks.
"""
assert self.dropout1_rate is None or 0 <= self.dropout1_rate < 1, 'Dropout rate 1 should be a value between 0 and 1 (value: {0})'.format(
self.dropout1_rate)
assert self.dropout2_rate is None or 0 <= self.dropout2_rate < 1, 'Dropout rate 2 should be a value between 0 and 1, or None for inheritance (value: {0})'.format(
self.dropout2_rate)
assert self.dropout3_rate is None or 0 <= self.dropout3_rate < 1, 'Dropout rate 3 should be a value between 0 and 1, or None for inheritance (value: {0})'.format(
self.dropout3_rate)
assert self.dense1_nonlinearity in nonlinearities.keys(
), 'Linearity 1 should be one of "{0}", got "{1}" instead.'.format(
'", "'.join(nonlinearities.keys()), self.dense1_nonlinearity)
assert self.dense2_nonlinearity in nonlinearities.keys() + [
None
], 'Linearity 2 should be one of "{0}", got "{1}" instead.'.format(
'", "'.join(nonlinearities.keys()), self.dense2_nonlinearity)
assert self.dense3_nonlinearity in nonlinearities.keys() + [
None
], 'Linearity 3 should be one of "{0}", got "{1}" instead.'.format(
'", "'.join(nonlinearities.keys()), self.dense3_nonlinearity)
assert self.dense1_init in initializers.keys(
), 'Initializer 1 should be one of "{0}", got "{1}" instead.'.format(
'", "'.join(initializers.keys()), self.dense1_init)
assert self.dense2_init in initializers.keys() + [
None
], 'Initializer 2 should be one of "{0}", got "{1}" instead.'.format(
'", "'.join(initializers.keys()), self.dense2_init)
assert self.dense3_init in initializers.keys() + [
None
], 'Initializer 3 should be one of "{0}", got "{1}" instead.'.format(
'", "'.join(initializers.keys()), self.dense3_init)
"""
Argument defaults.
"""
if self.dense2_nonlinearity is None:
self.dense2_nonlinearity = self.dense1_nonlinearity
if self.dense2_init is None:
self.dense2_init = self.dense1_init
if self.dense3_nonlinearity is None:
self.dense3_nonlinearity = self.dense2_nonlinearity
if self.dense3_init is None:
self.dense3_init = self.dense2_init
if self.dropout2_rate is None and self.dense2_size:
self.dropout2_rate = self.dropout1_rate
if self.dropout3_rate is None and self.dense3_size:
self.dropout3_rate = self.dropout2_rate
def fit(self, X, y, random_sleep=None):
if random_sleep:
sleep(random_sleep *
random()) # this is to prevent compiler lock problems
labels = y - y.min()
#todo: don't use labels.max(), occasionally (rarely) it will not have the highest class
self.init_net(feature_count=X.shape[1], class_count=labels.max() + 1)
net = self.net.fit(X, labels)
self.save()
return net
def interrupted_fit(self, X, y):
""" DEPRECATED """
labels = y - y.min()
self.init_net(feature_count=X.shape[1], class_count=labels.max() + 1)
knowledge = get_knowledge(self.net)
for epoch in range(0, self.max_epochs, self.epoch_steps):
set_knowledge(self.net, knowledge)
self.init_net(feature_count=X.shape[1],
class_count=labels.max() + 1)
print 'epoch {0:d}: learning {1:d} epochs'.format(
epoch, self.epoch_steps)
self.net.fit(X, labels)
ratio = mean([d['valid_loss'] for d in self.net._train_history[-self.epoch_steps:]]) / \
mean([d['train_loss'] for d in self.net._train_history[-self.epoch_steps:]])
if ratio < 0.85:
self.weight_decay *= 1.3
if ratio > 0.95:
self.weight_decay /= 1.2
self.init_net(feature_count=X.shape[1],
class_count=labels.max() + 1)
knowledge = get_knowledge(self.net)
exit()
net = self.net.fit(X, labels)
self.save()
return net
def predict_proba(self, X):
probs = self.net.predict_proba(X)
if not isfinite(probs).sum():
errmsg = 'network "{0:s}" predicted infinite/NaN probabilities'.format(
self.name)
stderr.write(errmsg)
raise DivergenceError(errmsg)
return probs
def predict(self, X):
return self.net.predict(X)
def score(self, X, y, **kwargs):
return self.net.score(X, y)
def save(self, filepath=None):
assert hasattr(
self, 'net'
), 'Cannot save a network that is not initialized; .fit(X, y) something first [or use net.initialize(..) for random initialization].'
parameters = self.get_params(deep=False)
filepath = filepath or join(NNET_STATE_DIR, self.name)
if VERBOSITY >= 1:
print 'saving network to "{0:s}.net.npz|json"'.format(filepath)
with open(filepath + '.net.json', 'w+') as fh:
dump([parameters, self.feature_count, self.class_count],
fp=fh,
indent=2)
save_knowledge(self.net, filepath + '.net.npz')
@classmethod
def load(cls, filepath=None, name=None):
"""
:param filepath: The base path (without extension) to load the file from, OR:
:param name: The name of the network to load (if filename is not given)
:return: The loaded network
"""
filepath = filepath or join(NNET_STATE_DIR, name)
if VERBOSITY >= 1:
print 'loading network from "{0:s}.net.npz|json"'.format(filepath)
with open(filepath + '.net.json', 'r') as fh:
[parameters, feature_count, class_count] = load(fp=fh)
nnet = cls(**parameters)
nnet.init_net(feature_count=feature_count, class_count=class_count)
load_knowledge(nnet.net, filepath + '.net.npz')
return nnet
class graspNet():
def __init__(self, param_file=None):
net_divider = 1.0
self.layers = [
(L.layers.InputLayer, {
'shape': (None, 7, X_H, X_W),
'name': 'input'
}),
(L.layers.Conv2DLayer, {
'num_filters': 96,
'stride': 1,
'pad': 3,
'filter_size': (7, 7),
'nonlinearity': L.nonlinearities.rectify,
'flip_filters': False,
'name': 'conv0'
}),
(L.layers.DropoutLayer, {
'p': 0.0
}),
(L.layers.MaxPool2DLayer, {
'pool_size': 3,
'ignore_border': False
}),
(L.layers.Conv2DLayer, {
'num_filters': 256,
'stride': 1,
'filter_size': (5, 5),
'nonlinearity': L.nonlinearities.rectify,
'flip_filters': False,
'name': 'conv1'
}),
(L.layers.DropoutLayer, {
'p': 0.0
}),
(L.layers.MaxPool2DLayer, {
'pool_size': 2,
'ignore_border': False
}),
(L.layers.Conv2DLayer, {
'num_filters': 512,
'stride': 1,
'pad': 1,
'filter_size': (3, 3),
'nonlinearity': L.nonlinearities.rectify,
'flip_filters': False,
'name': 'conv2'
}),
(L.layers.DropoutLayer, {
'p': 0.0
}),
(L.layers.Conv2DLayer, {
'num_filters': 512,
'stride': 1,
'pad': 1,
'filter_size': (3, 3),
'nonlinearity': L.nonlinearities.rectify,
'flip_filters': False,
'name': 'conv3'
}),
(L.layers.DropoutLayer, {
'p': 0.0
}),
(L.layers.Conv2DLayer, {
'num_filters': 512,
'stride': 1,
'pad': 1,
'filter_size': (3, 3),
'nonlinearity': L.nonlinearities.rectify,
'flip_filters': False,
'name': 'conv4'
}),
(L.layers.DropoutLayer, {
'p': 0.0
}),
(L.layers.MaxPool2DLayer, {
'pool_size': 3,
'ignore_border': False
}),
(L.layers.DenseLayer, {
'num_units': 4096,
'nonlinearity': L.nonlinearities.sigmoid,
'name': 'dense0'
}),
(L.layers.DropoutLayer, {
'p': 0.0
}),
(L.layers.DenseLayer, {
'num_units': 4096,
'nonlinearity': L.nonlinearities.sigmoid,
'name': 'dense1'
}),
(L.layers.DenseLayer, {
'num_units': 1,
'nonlinearity': L.nonlinearities.sigmoid
}),
]
self.net = NeuralNet(
layers=self.layers,
update_learning_rate=0.015,
update=L.updates.nesterov_momentum,
update_momentum=0.9,
#update=L.updates.sgd,
regression=True,
verbose=1,
eval_size=0.15,
objective_loss_function=L.objectives.binary_crossentropy,
max_epochs=200)
if param_file != None:
print "Loading parameters from ", param_file
self.net.load_params_from(param_file)
def write_filters_to_file(self, fname):
params = self.net.get_all_params()
layer_counter = 0
for p in params:
print p
layer_counter += 1
filter_counter = 0
weights = p.get_value()
if len(weights.shape) > 2:
for f in weights:
kernel = np.asarray(f, dtype=np.float32)
kernel = kernel * 255 + 128
viz = np.zeros(shape=kernel[0].shape)
viz = cv2.resize(kernel[0],
None,
fx=20,
fy=20,
interpolation=cv2.INTER_CUBIC)
cv2.normalize(viz, viz)
viz = viz * 50
#print viz
#cv2.imshow('ha', viz)
#cv2.waitKey(-1)
viz = viz / 50
viz = viz * 12000
viz = viz
#print viz
cv2.imwrite(
fname + "_" + str(layer_counter) + "_" +
str(filter_counter) + ".png", viz)
filter_counter += 1
# evaluates the input array over the neural net
def eval(self, x):
y = np.zeros((x.shape[0], ))
for i in range(x.shape[0]):
pred = self.net.predict(np.array([x[i]]))
y[i] = pred
print i, pred
return y
# train the network in input (x,y)
# param_file: input parameter file (perhaps from previous trainings)
# out_param_file: output path to write resulting params to
# filter_file: output path to write images of filters for visualization
# load_params: boolean flag, when set: 1 loads input parameters from <param_file>
# pretrain: boolean flag, when set: 1 loads parameters from pretrained network
def train(self,
X,
y,
param_file=None,
out_param_file=None,
filter_file=None,
load_params=False,
pretrain=False):
if pretrain:
params = createNoLearnParams(param_file)
print "Parameters", params[1].shape
conv0_W = np.concatenate((params[0], params[0]), axis=1)
conv0_W = np.concatenate((conv0_W, params[0]), axis=1)
conv0_W = conv0_W[:, :7, :, :]
conv0_b = np.concatenate((params[1], params[1]), axis=0)
conv0_b = np.concatenate((conv0_b, params[1]), axis=0)
conv0_b = conv0_b[:96]
conv1_W = np.concatenate((params[2], params[2]), axis=1)
conv1_W = np.concatenate((conv1_W, params[2]), axis=1)
conv1_W = conv1_W[:, :96, :, :]
conv1_b = np.concatenate((params[3], params[3]), axis=0)
conv1_b = np.concatenate((conv1_b, params[3]), axis=0)
conv1_b = conv1_b[:256]
conv2_W = np.concatenate((params[4], params[4]), axis=1)
conv2_W = np.concatenate((conv2_W, params[4]), axis=1)
conv2_W = conv2_W[:, :256, :, :]
conv2_b = np.concatenate((params[5], params[5]), axis=0)
conv2_b = np.concatenate((conv2_b, params[5]), axis=0)
conv2_b = conv2_b[:512]
conv3_W = np.concatenate((params[6], params[6]), axis=1)
conv3_W = np.concatenate((conv3_W, params[6]), axis=1)
conv3_W = conv3_W[:, :512, :, :]
conv3_b = np.concatenate((params[7], params[7]), axis=0)
conv3_b = np.concatenate((conv3_b, params[7]), axis=0)
conv3_b = conv3_b[:512]
conv4_W = np.concatenate((params[8], params[8]), axis=1)
conv4_W = np.concatenate((conv4_W, params[8]), axis=1)
conv4_W = conv4_W[:, :512, :, :]
conv4_b = np.concatenate((params[9], params[9]), axis=0)
conv4_b = np.concatenate((conv4_b, params[9]), axis=0)
conv4_b = conv4_b[:512]
dense0_W = np.concatenate((params[10], params[10]), axis=1)
dense0_W = np.concatenate((dense0_W, params[10]), axis=1)
dense0_W = dense0_W[:2560, :4096]
dense0_b = np.concatenate((params[11], params[11]), axis=0)
dense0_b = np.concatenate((dense0_b, params[11]), axis=0)
dense0_b = dense0_b[:4096]
dense1_W = np.concatenate((params[12], params[12]), axis=1)
dense1_W = np.concatenate((dense1_W, params[12]), axis=1)
dense1_W = dense1_W[:4096, :4096]
dense1_b = np.concatenate((params[13], params[13]), axis=0)
dense1_b = np.concatenate((dense1_b, params[13]), axis=0)
dense1_b = dense1_b[:4096]
#http://arxiv.org/pdf/1405.3531v4.pdf
self.net = NeuralNet(
layers=self.layers,
conv0_W=np.array(conv0_W),
conv0_b=np.array(conv0_b),
conv1_W=np.array(conv1_W),
conv1_b=np.array(conv1_b),
conv2_W=np.array(conv2_W),
conv2_b=np.array(conv2_b),
conv3_W=np.array(conv3_W),
conv3_b=np.array(conv3_b),
conv4_W=np.array(conv4_W),
conv4_b=np.array(conv4_b),
dense0_W=np.array(dense0_W),
dense0_b=np.array(dense0_b),
dense1_W=np.array(dense1_W),
dense1_b=np.array(dense1_b),
update_learning_rate=0.015,
update=L.updates.nesterov_momentum,
update_momentum=0.9,
#update=L.updates.sgd,
regression=True,
verbose=1,
eval_size=0.15,
objective_loss_function=L.objectives.binary_crossentropy,
max_epochs=200)
if load_params:
print "Loading parameters from ", param_file
self.net.load_params_from(param_file)
print "TRAINING!"
print "input shape: ", X.shape
print "output shape: ", y.shape
print "Example X", X[0]
print "Example Y", y[0]
#print self.net.get_params()
self.net.fit(X, y)
print(self.net.score(X, y))
print "Saving network parameters to ", out_param_file, "..."
file = open(out_param_file, 'w+')
file.close()
self.net.save_weights_to(out_param_file)
print "Saving filters to ", filter_file
self.write_filters_to_file(filter_file)
plt = visualize.plot_loss(self.net)
plt.show()
plt.savefig(DIR_PROJ + 'loss.png')
plt.clf()
plt.cla()
print "Sample predictions"
for i in range(10):
pred = self.net.predict(np.array([X[i]]))
print "---------------------------------"
print i
print "prediction", pred
print y[i]
def build_nn(df=None, class_column_name=None):
"""
Construct a classification neural network model from input dataframe
Parameters:
df : input dataframe
class_column_name : identity of the column in df with class data
"""
# Type check inputs for sanity
if df is None:
raise ValueError('df is None')
if not isinstance(df, pd.DataFrame):
raise TypeError('df is not a dataframe')
if class_column_name is None:
raise ValueError('class_column_name is None')
if not isinstance(class_column_name, basestring):
raise TypeError('class_column_name is not a string')
if class_column_name not in df.columns:
raise ValueError('class_column_name (%s) is not a valid column name'
% class_column_name)
df = df.sample(frac=1).reset_index(drop=True)
df_train, df_test = train_test_split(df, TEST_SIZE)
df_train, df_val = df_train[:(0.75 * len(df_train.index)), :], df_train[(0.75 * len(df_train.index)):, :]
x_train, x_val, x_test = df_train, df_val, df_test
# Remove the classification column from the dataframe
x_train = x_train.drop(class_column_name, axis=1, inplace=True).values
x_val = x_val.drop(class_column_name, axis=1, inplace=True).values
x_test = x_test.drop(class_column_name, axis=1, inplace=True).values
y_train = df_train[class_column_name].values.astype(np.int32)
y_val = df_val[class_column_name].values.astype(np.int32)
y_test = df_test[class_column_name].values.astype(np.int32)
# Create classification model
net = NeuralNet(layers=[('input', InputLayer),
('hidden0', DenseLayer),
('hidden1', DenseLayer),
('output', DenseLayer)],
input_shape=(None, x_train.shape[1]),
hidden0_num_units=NODES,
hidden0_nonlinearity=nonlinearities.softmax,
hidden1_num_units=NODES,
hidden1_nonlinearity=nonlinearities.softmax,
output_num_units=len(np.unique(y_train)),
output_nonlinearity=nonlinearities.softmax,
update_learning_rate=0.1,
verbose=1,
max_epochs=100)
param_grid = {'hidden0_num_units': [4, 17, 25],
'hidden0_nonlinearity':
[nonlinearities.sigmoid, nonlinearities.softmax],
'hidden1_num_units': [4, 17, 25],
'hidden1_nonlinearity':
[nonlinearities.sigmoid, nonlinearities.softmax],
'update_learning_rate': [0.01, 0.1, 0.5]}
grid_search = GridSearchCV(net, param_grid, verbose=0)
grid_search.fit(x_train, y_train)
net.fit(x_train, y_train)
print(net.score(x_train, y_train))
with open(PICKLE, 'wb') as file:
pickle.dump(x_train, file, pickle.HIGHEST_PROTOCOL)
pickle.dump(y_train, file, pickle.HIGHEST_PROTOCOL)
pickle.dump(df_test, file, pickle.HIGHEST_PROTOCOL)
pickle.dump(grid_search, file, pickle.HIGHEST_PROTOCOL)
pickle.dump(net, file, pickle.HIGHEST_PROTOCOL)
def build_net(train, test, y_scaler):
xs_test, ys_test = test
xs_train, ys_train = train
num_features = xs_train.shape[1]
#assert(num_features == len(feature_extractions().keys()))
loss_function = get_loss_function(y_scaler)
input_var = theano.tensor.dmatrix('inputs')
target_var = theano.tensor.dvector('targets')
# l_in = las.layers.InputLayer((len(xs_test), len(xs_test[0])), input_var=input_var)
# l_recur_a = las.layers.RecurrentLayer(l_in, num_units= 50)
# l_hidden = las.layers.DenseLayer(l_recur_a, num_units = 4,nonlinearity = las.nonlinearities.softmax, W=las.init.Normal(0.1))
# l_recur_b = las.layers.RecurrentLayer(l_hidden, num_units = 4) #Try doing custom
# -----pure classes below
c_l_in = las.layers.InputLayer
c_l_recur_a = las.layers.RecurrentLayer
c_l_hidden = las.layers.DenseLayer
c_l_recur_b = las.layers.RecurrentLayer #Try doing custom
c_expression_layer = las.layers.special.ExpressionLayer
c_output = las.layers.DenseLayer
#layers = [('input', c_l_in), ('a', c_l_recur_a), ('h', c_l_hidden), ('b', c_l_recur_b),('output', c_output)]
layers = [('input', c_l_in), ('h', c_l_hidden), ('h2', c_l_hidden),('h3', c_l_hidden),('h4', c_l_hidden), ('output', c_output)]
print "\nBuilding..."
#o = binary_hinge_loss
net0 = NeuralNet(layers=layers,
regression=True,
y_tensor_type=theano.tensor.type.TensorType('float64', (False, True)) ,
input_shape=(None, num_features),
# input_input_var = input_var,
# a_num_units = 50,
h_num_units=400,
#h_nonlinearity = las.nonlinearities.softmax,
h2_num_units=50,
h3_num_units=20,
h4_num_units=1,
# h2_nonlinearity = las.nonlinearities.softmax,
# b_num_units = 4,
#e_function=expression_layer_fn,
output_num_units=1,
# output_nonlinearity=softmax,
objective_loss_function=loss_function,
update=nesterov_momentum,
update_learning_rate=0.001,
update_momentum=0.3,
train_split=nolearn.lasagne.TrainSplit(eval_size=0.1),
verbose=1,
max_epochs=1000)
print "Begin training"
net0.fit(xs_train, ys_train)
print "y: %f" % (ys_test[0])
print "transformed y: %f" %(y_scaler.inverse_transform([ys_test[0]])[0])
print "\n"
print "y: %f" % (ys_test[1])
print "transformed y: %f" % (y_scaler.inverse_transform([ys_test[1]])[0])
print "\n predictions: :"
print "y: {}".format((net0.predict([xs_test[0], xs_test[1]])))
print y_scaler.inverse_transform(net0.predict([xs_test[0], xs_test[1]]))
# predicts = net0.predict([[30.0,-1.5,4.5,3087],[1.0,1.0,1.0,1.0],[5.0,0.1,5.0,1000]])
# print "\nPrediction: %f - 93864 == %f \n %f - 3 == %f \n %f - 1000000 == %f" % (predicts[0], (predicts[0]-93864)*1.0/93864, predicts[1], (predicts[1] - 3)*1.0/3, predicts[2], (predicts[2]-1000000)/1000000)
# print "\n\nTransformed:"
# predicts = map(lambda x: y_scaler.inverse_transform([x]), predicts)
# predicts = [y[0] for y in predicts]
# print "prediction: %f - 93864 == %f \n %f - 3 == %f \n %f - 1000000 == %f" % (predicts[0], (predicts[0]-93864)*1.0/93864, predicts[1], (predicts[1] - 3)*1.0/3, predicts[2], (predicts[2]-1000000)/1000000)
print "\n Scores:"
print "test score: %f" % (net0.score(xs_test,ys_test))
print net0.score(xs_train,ys_train)
print "random score: %f" % (net0.score(xs_test,ys_train[0:len(xs_test)]))
print "random score: %f" % (net0.score(xs_train[0:len(ys_test)],ys_test))
print net0.layers
(DenseLayer,
dict(name='out',
num_units=n_classes,
nonlinearity=nonlinearities.softmax)),
],
regression=False,
objective_loss_function=objectives.categorical_crossentropy,
update=updates.adam,
batch_iterator_train=train_iterator,
batch_iterator_test=test_iterator,
on_epoch_finished=[
save_weights, save_training_history, plot_training_history
],
verbose=10,
max_epochs=250,
)
if __name__ == '__main__':
# X_train, X_test are image file names
# They will be read in the iterator
X_train, X_test, y_train, y_test = load_data(test_size=0.25,
random_state=42)
net.fit(X_train, y_train)
# Load the best weights from pickled model
net.load_params_from('./examples/cifar10/model_weights.pkl')
score = net.score(X_test, y_test)
print 'Final score %.4f' % score
NN0 = NeuralNet(layers = layer0,
max_epochs = 10,
# optimization method:
update=adam,
update_learning_rate=0.0002
)
# In[159]:
NN0.fit(x_train, y_train)
# In[160]:
NN0.score(x_vali, y_vali)
# the accuracy is only 0.95, lower than the score of random forest of 0.966
# # model 1: one input layer, two hidden layers, and one output layer
# In[172]:
layer1=[(layers.InputLayer, {'shape': (None, 1, 28, 28)}),
(layers.DenseLayer, {'num_units':1000}),
(layers.DropoutLayer, {}),
(layers.DenseLayer, {'num_units':1000}),
(layers.DenseLayer, {'num_units':10, 'nonlinearity': softmax})]
# In[173]:
def smart_find(X, y, X_valid, y_valid):
loss = []
kf = KFold(n_splits=5, shuffle=True)
conf_set = set()
step = (64 + 10) / 4
max_neuron_units = step * 8
for i in range(1, max_neuron_units, step):
for j in range(0, max_neuron_units, step):
for k in range(0, max_neuron_units, step):
struct_net = (i)
l = InputLayer(shape=(None, X.shape[1]))
# ------- HIDDEN -----------
l = DenseLayer(l, num_units=i, nonlinearity=softmax)
if j > 0:
if i + step < j:
continue
l = DenseLayer(l, num_units=j, nonlinearity=softmax)
struct_net = (i, j)
if k > 0:
if i + step < k or j + step < k:
continue
struct_net = (i, j, k)
l = DenseLayer(l, num_units=k, nonlinearity=softmax)
# ------- HIDDEN -----------
l = DenseLayer(l,
num_units=len(np.unique(y)),
nonlinearity=softmax)
net = NeuralNet(l,
update=adam,
update_learning_rate=0.01,
max_epochs=250)
if struct_net in conf_set:
continue
print('=' * 40)
print(struct_net)
print('=' * 40)
conf_set.add(struct_net)
k_loss = []
y_data = np.array([y]).transpose()
data = np.concatenate((X, y_data), axis=1)
for train_index, test_index in kf.split(data):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
net.fit(X_train, y_train)
y_pred = net.predict(X_test)
loss_error = net.score(X_test, y_test)
# loss_error = mean_squared_error(y_test, y_pred)
k_loss.append(loss_error)
print(loss_error)
loss_net = (i, j, k, np.array(k_loss).mean())
print(loss_net)
loss.append(loss_net)
print('=' * 40)
# for i in range(1, max_hidden_layers):
# for j in range((64 + 10) // 2 // i, max_neuron_units // i, 10 // i):
# print('=' * 40)
# print('%s hidden layers' % i)
# print('%s neurons' % j)
# print('=' * 40)
# l = InputLayer(shape=(None, X.shape[1]))
# for k in range(i):
# l = DenseLayer(l, num_units=j, nonlinearity=softmax)
# l = DenseLayer(l, num_units=len(np.unique(y)), nonlinearity=softmax)
# net = NeuralNet(l, update=adam, update_learning_rate=0.01, max_epochs=500)
#
# k_loss = []
# y_data = np.array([y]).transpose()
# data = np.concatenate((X, y_data), axis=1)
# for train_index, test_index in kf.split(data):
# X_train, X_test = X[train_index], X[test_index]
# y_train, y_test = y[train_index], y[test_index]
#
# net.fit(X_train, y_train)
# y_pred = net.predict(X_test)
# loss_error = mean_squared_error(y_test, y_pred)
# k_loss.append(loss_error)
# print(loss_error)
#
# loss_net = (i, j, np.array(k_loss).mean())
# print(loss_net)
# loss.append(loss_net)
# print('=' * 40)
print(min(loss, key=lambda x: x[3]))
print(max(loss, key=lambda x: x[3]))
print(loss)
print "X_training shape must match y_training shape"
print "Generate X_test and y_test"
n_input = 11
print "X_test..."
print "Multi Layer Perceptron..."
#Build layer for MLP
l_in = ls.layers.InputLayer(shape=(None,10),input_var=None)
l_hidden = ls.layers.DenseLayer(l_in,num_units=15,nonlinearity=ls.nonlinearities.sigmoid)
network = l_out = ls.layers.DenseLayer(l_hidden,num_units=1)
print "Neural network initialize"
#Init Neural net
net1 = NeuralNet(
layers=network,
# optimization method:
update=nesterov_momentum,
update_learning_rate=0.001,
update_momentum=0.9,
regression=True, # flag to indicate we're dealing with regression problem
max_epochs=400, # we want to train this many epochs
verbose=1,
)
#
print "Training time!!!!!....."
net1.fit(X_training,y_training)
net1.save_params_to("saveNeuralNetwork.tdn")
print "Score rate = "
print net1.score(n_sample2,n_test2)
print net1.predict(n_sample2)[0:2]
(FeaturePoolLayer, dict(name='l8p', pool_size=2)),
(DropoutLayer, dict(name='l8drop', p=0.5)),
(DenseLayer, dict(name='out', num_units=10, nonlinearity=nonlinearities.softmax)),
],
regression=False,
objective_loss_function=objectives.categorical_crossentropy,
update=updates.adam,
update_learning_rate=1e-3,
batch_iterator_train=train_iterator,
batch_iterator_test=test_iterator,
on_epoch_finished=[
save_training_history,
plot_training_history,
],
verbose=10,
max_epochs=20
)
if __name__ == '__main__':
X_train, X_test, y_train, y_test = load_data(test_size=0.25, random_state=42)
print "Training Network"
net.fit(X_train, y_train)
score = net.score(X_test, y_test)
print 'Final score %.4f' % score
