def gridsearch_alpha(self,learning_rate,index,params=None): hidden_unit = ((index+1)*2)/3 self.l_in = ls.layers.InputLayer(shape=(None,n_input),input_var=None) self.l_hidden = ls.layers.DenseLayer(self.l_in,num_units=15,nonlinearity=ls.nonlinearities.rectify) self.network = l_out = ls.layers.DenseLayer(self.l_hidden,num_units=1) list_results = np.array([learning_rate.shape[0]],dtype=np.float64) for item in learning_rate: #Init Neural net net1 = NeuralNet( layers=self.network, # optimization method: update=nesterov_momentum, update_learning_rate=item, update_momentum=0.9, regression=True, # flag to indicate we're dealing with regression problem max_epochs=800, # we want to train this many epochs # verbose=1, eval_size = 0.4 ) net1.fit(self.X_training,self.y_training) self.pred = net1.predict(self.n_sample2) name_file = "Params/saveNeuralNetwork_%s_%s.tdn" %(item,index) net1.save_params_to(name_file) score_nn = net1.score(self.n_sample2,self.n_test2) list_results[item] = score_nn print "index=%f,item=%f,score=%f"%(index,item,score_nn) return list_results
def gridsearch_alpha(self, learning_rate, index, params=None): self.l_in = ls.layers.InputLayer(shape=(None, n_input), input_var=None, W=params.T) self.l_hidden = ls.layers.DenseLayer( self.l_in, num_units=15, nonlinearity=ls.nonlinearities.rectify) self.network = l_out = ls.layers.DenseLayer(self.l_hidden, num_units=1) list_results = np.array([learning_rate.shape[0]], dtype=np.float64) for item in learning_rate: #Init Neural net net1 = NeuralNet( layers=self.network, # optimization method: update=nesterov_momentum, update_learning_rate=item, update_momentum=0.9, regression= True, # flag to indicate we're dealing with regression problem max_epochs=800, # we want to train this many epochs # verbose=1, eval_size=0.4) # net1.fit(self.X_training, self.y_training) self.pred = net1.predict(self.n_sample2) name_file = "GeneticParams/saveNeuralNetwork_%s_%s.tdn" % (item, index) net1.save_params_to(name_file) score_nn = net1.score(self.n_sample2, self.n_test2) list_results[item] = score_nn print "index=%s,item=%f,score=%f" % (index, item, score_nn) return list_results
def train_nn_model(): imageSize = 400 # 20 x 20 pixels from lasagne import layers from lasagne.updates import nesterov_momentum, sgd from nolearn.lasagne import NeuralNet model = NeuralNet(layers=[('input', layers.InputLayer), ('hidden', layers.DenseLayer), ('output', layers.DenseLayer),], # layer parameters: input_shape=(None, 400), # 96x96 input pixels per batch hidden_num_units=100, # number of units in hidden layer output_nonlinearity=None, # output layer uses identity function output_num_units=62, # 30 target values # optimization method: update=nesterov_momentum, update_learning_rate=0.01, update_momentum=0.9, regression=True, # flag to indicate we're dealing with regression problem max_epochs=400, # we want to train this many epochs verbose=1,) xTrain, yTrain, xTest, labelsInfoTest = load_train_test_data(nn_ytrain=True) print xTrain.shape, yTrain.shape print xTrain.dtype, yTrain.dtype model.fit(xTrain, yTrain) ytest_pred = model.predict(xTrain) print model.score(xTrain, yTrain) print accuracy_score(ytest_pred, yTrain) yTest = model.predict(xTest) print labelsInfoTest.shape, yTest.shape yTest2 = transform_to_class(yTest) submit_df = labelsInfoTest submit_df['Class'] = yTest2 submit_df.to_csv('submission.csv', index=False) return model
def classify(X, y): l = InputLayer(shape=(None, X.shape[1])) l = DenseLayer(l, num_units=100, nonlinearity=tanh) l = DenseLayer(l, num_units=len(np.unique(y)), nonlinearity=softmax) net = NeuralNet(l, update_learning_rate=0.05, train_split=TrainSplit(eval_size=0.2), verbose=1) net.fit(X, y) print(net.score(X, y))
def regr(X, Y): l = InputLayer(shape=(None, X.shape[1])) l = DenseLayer(l, num_units=X.shape[1], nonlinearity=tanh) #tanh, sigmoid # l = DropoutLayer(l, p=0.3, rescale=True) # previous: p=0.5 l = DenseLayer(l, num_units=1, nonlinearity=sigmoid) # l = DropoutLayer(l, p=0.3, rescale=True) # previous: p=0.5 net = NeuralNet(l, regression=True, update_learning_rate=0.01, verbose=1, max_epochs=700) net.fit(X, Y) print(net.score(X, Y)) return net
def regress(X, y): l = InputLayer(shape=(None, X.shape[1])) l = DenseLayer(l, num_units=100, nonlinearity=tanh) l = DenseLayer(l, num_units=y.shape[1], nonlinearity=None) net = NeuralNet(l, regression=True, update_learning_rate=0.05, train_split=TrainSplit(eval_size=0.2), verbose=1) net.fit(X, y) print(net.score(X, y))
def regr(X, Y): l = InputLayer(shape=(None, X.shape[1])) l = DenseLayer(l, num_units=Y.shape[1]+100, nonlinearity=tanh) # l = DropoutLayer(l, p=0.3, rescale=True) # previous: p=0.5 l = DenseLayer(l, num_units=Y.shape[1]+50, nonlinearity=tanh) # l = DropoutLayer(l, p=0.3, rescale=True) # previous: p=0.5 l = DenseLayer(l, num_units=Y.shape[1], nonlinearity=None) net = NeuralNet(l, regression=True, update_learning_rate=0.1, verbose=1) net.fit(X, Y) print(net.score(X, Y)) return net
def cancer(X, y, X_valid, y_valid): l = InputLayer(shape=(None, X.shape[1])) l = DenseLayer(l, num_units=len(np.unique(y)), nonlinearity=softmax) net = NeuralNet(l, update_learning_rate=0.01, max_epochs=1000) net.fit(X, y) print(net.score(X, y)) y_pred = net.predict(X_valid) print(y_valid) print(y_pred) plot_loss(net) plt.title('Cancer') plt.show()
def regr(X, y, X_valid, y_valid): l = InputLayer(shape=(None, X.shape[1])) l = DenseLayer(l, num_units=100, nonlinearity=softmax) # l = DenseLayer(l, num_units=40, nonlinearity=softmax) l = DenseLayer(l, num_units=len(np.unique(y)), nonlinearity=softmax) net = NeuralNet(l, update=adam, update_learning_rate=0.01, max_epochs=2000, objective_loss_function=squared_error, regression=True) net.fit(X, y) print(net.score(X, y)) y_pred = net.predict(X_valid) print(y_valid) print(y_pred) plot_loss(net) plt.title('Digits') plt.show()
def test_lasagne_functional_regression(boston): from nolearn.lasagne import NeuralNet X, y = boston layer1 = InputLayer(shape=(128, 13)) layer2 = DenseLayer(layer1, num_units=100) output = DenseLayer(layer2, num_units=1, nonlinearity=identity) nn = NeuralNet( layers=output, update_learning_rate=0.01, update_momentum=0.1, regression=True, max_epochs=50, ) nn.fit(X[:300], y[:300]) assert mean_absolute_error(nn.predict(X[300:]), y[300:]) < 3.0 assert r2_score(nn.predict(X[300:]), y[300:]) == nn.score(X[300:], y[300:])
def classify(X, y, X_test, y_test): layers0 = [('input', InputLayer), ('dense0', DenseLayer), ('dropout0', DropoutLayer), ('dense1', DenseLayer), ('dropout1', DropoutLayer), ('output', DenseLayer)] net = NeuralNet(layers=layers0, input_shape=(None, X.shape[1]), dense0_num_units=300, dropout0_p=0.075, dropout1_p=0.1, dense1_num_units=750, output_num_units=3, output_nonlinearity=softmax, update=nesterov_momentum, update_learning_rate=0.001, update_momentum=0.99, eval_size=0.2, verbose=1, max_epochs=15) net.fit(X, y) print(net.score(X, y)) preds = net.predict(X_test) print(classification_report(y_test, preds)) cm = confusion_matrix(y_test, preds) plt.matshow(cm) plt.title('Confusion matrix') plt.colorbar() plt.ylabel('True label') plt.xlabel('Predicted label') plt.savefig('confmatrix.png') plt.show() print(cm)
def train_nolearn_model(X, y): ''' NeuralNet with nolearn ''' X = X.astype(np.float32) y = y.astype(np.int32) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=5) X_train, X_test = impute_nan(X_train, X_test) X_train, X_test = normalize_features(X_train, X_test) lays = [ ('input', layers.InputLayer), ('hidden', layers.DenseLayer), ('output', layers.DenseLayer), ] net = NeuralNet( layers=lays, input_shape=(None, 23), hidden_num_units=10, objective_loss_function=lasagne.objectives.categorical_crossentropy, output_nonlinearity=lasagne.nonlinearities.sigmoid, output_num_units=10, update=nesterov_momentum, update_learning_rate=0.001, update_momentum=0.9, max_epochs=10, verbose=1, ) #net.fit(X_train, y_train) #predicted = net.predict(X_test) test_score = net.predict(X_test, y_test) train_score = net.score(X_train, y_train) return train_score, test_score
def train_nolearn_model(X, y): ''' NeuralNet with nolearn ''' X = X.astype(np.float32) y = y.astype(np.int32) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 5) X_train, X_test = impute_nan(X_train, X_test) X_train, X_test = normalize_features(X_train, X_test) lays = [('input', layers.InputLayer), ('hidden', layers.DenseLayer), ('output', layers.DenseLayer), ] net = NeuralNet( layers = lays, input_shape=(None, 23), hidden_num_units=10, objective_loss_function=lasagne.objectives.categorical_crossentropy, output_nonlinearity=lasagne.nonlinearities.sigmoid, output_num_units=10, update = nesterov_momentum, update_learning_rate= 0.001, update_momentum=0.9, max_epochs=10, verbose=1, ) #net.fit(X_train, y_train) #predicted = net.predict(X_test) test_score = net.predict(X_test, y_test) train_score = net.score(X_train, y_train) return train_score, test_score
def build_nn(df=None, class_column_name=None): """ Construct a classification neural network model from input dataframe Parameters: df : input dataframe class_column_name : identity of the column in df with class data """ # Type check inputs for sanity if df is None: raise ValueError('df is None') if not isinstance(df, pd.DataFrame): raise TypeError('df is not a dataframe') if class_column_name is None: raise ValueError('class_column_name is None') if not isinstance(class_column_name, basestring): raise TypeError('class_column_name is not a string') if class_column_name not in df.columns: raise ValueError('class_column_name (%s) is not a valid column name' % class_column_name) df = df.sample(frac=1).reset_index(drop=True) df_train, df_test = train_test_split(df, TEST_SIZE) df_train, df_val = df_train[:(0.75 * len(df_train.index)), :], df_train[( 0.75 * len(df_train.index)):, :] x_train, x_val, x_test = df_train, df_val, df_test # Remove the classification column from the dataframe x_train = x_train.drop(class_column_name, axis=1, inplace=True).values x_val = x_val.drop(class_column_name, axis=1, inplace=True).values x_test = x_test.drop(class_column_name, axis=1, inplace=True).values y_train = df_train[class_column_name].values.astype(np.int32) y_val = df_val[class_column_name].values.astype(np.int32) y_test = df_test[class_column_name].values.astype(np.int32) # Create classification model net = NeuralNet(layers=[('input', InputLayer), ('hidden0', DenseLayer), ('hidden1', DenseLayer), ('output', DenseLayer)], input_shape=(None, x_train.shape[1]), hidden0_num_units=NODES, hidden0_nonlinearity=nonlinearities.softmax, hidden1_num_units=NODES, hidden1_nonlinearity=nonlinearities.softmax, output_num_units=len(np.unique(y_train)), output_nonlinearity=nonlinearities.softmax, update_learning_rate=0.1, verbose=1, max_epochs=100) param_grid = { 'hidden0_num_units': [4, 17, 25], 'hidden0_nonlinearity': [nonlinearities.sigmoid, nonlinearities.softmax], 'hidden1_num_units': [4, 17, 25], 'hidden1_nonlinearity': [nonlinearities.sigmoid, nonlinearities.softmax], 'update_learning_rate': [0.01, 0.1, 0.5] } grid_search = GridSearchCV(net, param_grid, verbose=0) grid_search.fit(x_train, y_train) net.fit(x_train, y_train) print(net.score(x_train, y_train)) with open(PICKLE, 'wb') as file: pickle.dump(x_train, file, pickle.HIGHEST_PROTOCOL) pickle.dump(y_train, file, pickle.HIGHEST_PROTOCOL) pickle.dump(df_test, file, pickle.HIGHEST_PROTOCOL) pickle.dump(grid_search, file, pickle.HIGHEST_PROTOCOL) pickle.dump(net, file, pickle.HIGHEST_PROTOCOL)
def classif(X, y): l = InputLayer(shape=(None, X.shape[1])) l = DenseLayer(l, num_units=len(np.unique(y)), nonlinearity=softmax) net = NeuralNet(l, update_learning_rate=0.01) net.fit(X, y) print(net.score(X, y))
def regr(X, y): l = InputLayer(shape=(None, X.shape[1])) l = DenseLayer(l, num_units=y.shape[1], nonlinearity=None) net = NeuralNet(l, regression=True, update_learning_rate=0.01) net.fit(X, y) print(net.score(X, y))
print 'started', datetime.now() res = net0.fit(features_scaled, labels.astype('int32')) print 'finished', datetime.now() print res # In[2]: import pickle pickle.dump(net0, open('lasagne_model.pkl', 'wb')) # In[106]: print 'score', net0.score(features_scaled, labels) # In[111]: print 'logloss', log_loss(labels, net0.predict_proba(features_scaled)) # In[16]: test_features, _ = get_features(test) scaler.fit(test_features) test_features = scaler.transform(test_features)
class NNet(BaseEstimator, ClassifierMixin): def __init__( self, name='nameless_net', # used for saving, so maybe make it unique dense1_size=60, dense1_nonlinearity='tanh', dense1_init='orthogonal', dense2_size=None, dense2_nonlinearity=None, # inherits dense1 dense2_init=None, # inherits dense1 dense3_size=None, dense3_nonlinearity=None, # inherits dense2 dense3_init=None, # inherits dense2 learning_rate=0.001, learning_rate_scaling=100, momentum=0.9, momentum_scaling=100, max_epochs=3000, epoch_steps=None, dropout0_rate=0, # this is the input layer dropout1_rate=None, dropout2_rate=None, # inherits dropout1_rate dropout3_rate=None, # inherits dropout2_rate weight_decay=0, adaptive_weight_decay=False, batch_size=128, output_nonlinearity='softmax', auto_stopping=True, save_snapshots_stepsize=None, ): """ Create the network with the selected parameters. :param name: Name for save files :param dense1_size: Number of neurons for first hidden layer :param dense1_nonlinearity: The activation function for the first hidden layer :param dense1_init: The weight initialization for the first hidden layer :param learning_rate: The (initial) learning rate (how fast the network learns) :param learning_rate_scaling: The total factor to gradually decrease the learning rate by :param momentum: The (initial) momentum :param momentum_scaling: Similar to learning_rate_scaling :param max_epochs: Total number of epochs (at most) :param dropout1_rate: Percentage of connections dropped each step for first hidden layer :param weight_decay: Palatalizes the weights by L2 norm (regularizes but decreases results) :param adaptive_weight_decay: Should the weight decay adapt automatically? :param batch_size: How many samples to send through the network at a time :param auto_stopping: Stop early if the network seems to stop performing well :param pretrain: Filepath of the previous weights to start at (or None) :return: """ """ Input argument storage: automatically store all locals, which should be exactly the arguments at this point, but storing a little too much is not a big problem. """ params = locals() del params['self'] #self.__dict__.update(params) self.parameter_names = sorted(params.keys()) """ Check the parameters and update some defaults (will be done for 'self', no need to store again). """ self.set_params(**params) def init_net(self, feature_count, class_count=NCLASSES, verbosity=VERBOSITY >= 2): """ Initialize the network (needs to be done when data is available in order to set dimensions). """ if VERBOSITY >= 1: print 'initializing network {0:s} {1:d}x{2:d}x{3:d}'.format( self.name, self.dense1_size or 0, self.dense2_size or 0, self.dense3_size or 0) if VERBOSITY >= 2: print 'parameters: ' + ', '.join( '{0:s} = {1:}'.format(k, v) for k, v in self.get_params(deep=False).items()) self.feature_count = feature_count self.class_count = class_count """ Create the layers and their settings. """ self.layers = [ ('input', InputLayer), ] self.params = { 'dense1_num_units': self.dense1_size, 'dense1_nonlinearity': nonlinearities[self.dense1_nonlinearity], 'dense1_W': initializers[self.dense1_init], 'dense1_b': Constant(0.), } if self.dropout0_rate: self.layers += [('dropout0', DropoutLayer)] self.params['dropout0_p'] = self.dropout0_rate self.layers += [ ('dense1', DenseLayer), ] if self.dropout1_rate: self.layers += [('dropout1', DropoutLayer)] self.params['dropout1_p'] = self.dropout1_rate if self.dense2_size: self.layers += [('dense2', DenseLayer)] self.params.update({ 'dense2_num_units': self.dense2_size, 'dense2_nonlinearity': nonlinearities[self.dense2_nonlinearity], 'dense2_W': initializers[self.dense2_init], 'dense2_b': Constant(0.), }) else: assert not self.dense3_size, 'There cannot be a third dense layer without a second one' if self.dropout2_rate: assert self.dense2_size is not None, 'There cannot be a second dropout layer without a second dense layer.' self.layers += [('dropout2', DropoutLayer)] self.params['dropout2_p'] = self.dropout2_rate if self.dense3_size: self.layers += [('dense3', DenseLayer)] self.params.update({ 'dense3_num_units': self.dense3_size, 'dense3_nonlinearity': nonlinearities[self.dense3_nonlinearity], 'dense3_W': initializers[self.dense3_init], 'dense3_b': Constant(0.), }) if self.dropout3_rate: assert self.dense2_size is not None, 'There cannot be a third dropout layer without a third dense layer.' self.layers += [('dropout3', DropoutLayer)] self.params['dropout3_p'] = self.dropout2_rate self.layers += [('output', DenseLayer)] self.params.update({ 'output_nonlinearity': nonlinearities[self.output_nonlinearity], 'output_W': GlorotUniform(), 'output_b': Constant(0.), }) """ Create meta parameters and special handlers. """ if VERBOSITY >= 3: print 'learning rate: {0:.6f} -> {1:.6f}'.format( abs(self.learning_rate), abs(self.learning_rate) / float(self.learning_rate_scaling)) print 'momentum: {0:.6f} -> {1:.6f}'.format( abs(self.momentum), 1 - ((1 - abs(self.momentum)) / float(self.momentum_scaling))) self.step_handlers = [ LinearVariable('update_learning_rate', start=abs(self.learning_rate), stop=abs(self.learning_rate) / float(self.learning_rate_scaling)), LinearVariable( 'update_momentum', start=abs(self.momentum), stop=1 - ((1 - abs(self.momentum)) / float(self.momentum_scaling))), StopNaN(), ] self.end_handlers = [ SnapshotEndSaver(base_name=self.name), TrainProgressPlotter(base_name=self.name), ] snapshot_name = 'nn_' + params_name(self.params, prefix=self.name)[0] if self.save_snapshots_stepsize: self.step_handlers += [ SnapshotStepSaver(every=self.save_snapshots_stepsize, base_name=snapshot_name), ] if self.auto_stopping: self.step_handlers += [ StopWhenOverfitting(loss_fraction=0.9, base_name=snapshot_name), StopAfterMinimum(patience=40, base_name=self.name), ] weight_decay = shared(float32(abs(self.weight_decay)), 'weight_decay') if self.adaptive_weight_decay: self.step_handlers += [ AdaptiveWeightDecay(weight_decay), ] if self.epoch_steps: self.step_handlers += [ BreakEveryN(self.epoch_steps), ] """ Create the actual nolearn network with information from __init__. """ self.net = NeuralNet( layers=self.layers, objective=partial(WeightDecayObjective, weight_decay=weight_decay), input_shape=(None, feature_count), output_num_units=class_count, update=nesterov_momentum, # todo: make parameter update_learning_rate=shared(float32(self.learning_rate)), update_momentum=shared(float(self.weight_decay)), on_epoch_finished=self.step_handlers, on_training_finished=self.end_handlers, regression=False, max_epochs=self.max_epochs, verbose=verbosity, batch_iterator_train=BatchIterator(batch_size=self.batch_size), batch_iterator_test=BatchIterator(batch_size=self.batch_size), eval_size=0.1, #custom_score = ('custom_loss', categorical_crossentropy), **self.params) self.net.parent = self self.net.initialize() return self.net def get_params(self, deep=True): return OrderedDict( (name, getattr(self, name)) for name in self.parameter_names) def set_params(self, **params): """ Set all the parameters. """ for name, val in params.items(): assert name in self.parameter_names, '"{0:s}" is not a valid parameter name (known parameters: "{1:s}")'.format( name, '", "'.join(self.parameter_names)) setattr(self, name, val) """ Arguments checks. """ assert self.dropout1_rate is None or 0 <= self.dropout1_rate < 1, 'Dropout rate 1 should be a value between 0 and 1 (value: {0})'.format( self.dropout1_rate) assert self.dropout2_rate is None or 0 <= self.dropout2_rate < 1, 'Dropout rate 2 should be a value between 0 and 1, or None for inheritance (value: {0})'.format( self.dropout2_rate) assert self.dropout3_rate is None or 0 <= self.dropout3_rate < 1, 'Dropout rate 3 should be a value between 0 and 1, or None for inheritance (value: {0})'.format( self.dropout3_rate) assert self.dense1_nonlinearity in nonlinearities.keys( ), 'Linearity 1 should be one of "{0}", got "{1}" instead.'.format( '", "'.join(nonlinearities.keys()), self.dense1_nonlinearity) assert self.dense2_nonlinearity in nonlinearities.keys() + [ None ], 'Linearity 2 should be one of "{0}", got "{1}" instead.'.format( '", "'.join(nonlinearities.keys()), self.dense2_nonlinearity) assert self.dense3_nonlinearity in nonlinearities.keys() + [ None ], 'Linearity 3 should be one of "{0}", got "{1}" instead.'.format( '", "'.join(nonlinearities.keys()), self.dense3_nonlinearity) assert self.dense1_init in initializers.keys( ), 'Initializer 1 should be one of "{0}", got "{1}" instead.'.format( '", "'.join(initializers.keys()), self.dense1_init) assert self.dense2_init in initializers.keys() + [ None ], 'Initializer 2 should be one of "{0}", got "{1}" instead.'.format( '", "'.join(initializers.keys()), self.dense2_init) assert self.dense3_init in initializers.keys() + [ None ], 'Initializer 3 should be one of "{0}", got "{1}" instead.'.format( '", "'.join(initializers.keys()), self.dense3_init) """ Argument defaults. """ if self.dense2_nonlinearity is None: self.dense2_nonlinearity = self.dense1_nonlinearity if self.dense2_init is None: self.dense2_init = self.dense1_init if self.dense3_nonlinearity is None: self.dense3_nonlinearity = self.dense2_nonlinearity if self.dense3_init is None: self.dense3_init = self.dense2_init if self.dropout2_rate is None and self.dense2_size: self.dropout2_rate = self.dropout1_rate if self.dropout3_rate is None and self.dense3_size: self.dropout3_rate = self.dropout2_rate def fit(self, X, y, random_sleep=None): if random_sleep: sleep(random_sleep * random()) # this is to prevent compiler lock problems labels = y - y.min() #todo: don't use labels.max(), occasionally (rarely) it will not have the highest class self.init_net(feature_count=X.shape[1], class_count=labels.max() + 1) net = self.net.fit(X, labels) self.save() return net def interrupted_fit(self, X, y): """ DEPRECATED """ labels = y - y.min() self.init_net(feature_count=X.shape[1], class_count=labels.max() + 1) knowledge = get_knowledge(self.net) for epoch in range(0, self.max_epochs, self.epoch_steps): set_knowledge(self.net, knowledge) self.init_net(feature_count=X.shape[1], class_count=labels.max() + 1) print 'epoch {0:d}: learning {1:d} epochs'.format( epoch, self.epoch_steps) self.net.fit(X, labels) ratio = mean([d['valid_loss'] for d in self.net._train_history[-self.epoch_steps:]]) / \ mean([d['train_loss'] for d in self.net._train_history[-self.epoch_steps:]]) if ratio < 0.85: self.weight_decay *= 1.3 if ratio > 0.95: self.weight_decay /= 1.2 self.init_net(feature_count=X.shape[1], class_count=labels.max() + 1) knowledge = get_knowledge(self.net) exit() net = self.net.fit(X, labels) self.save() return net def predict_proba(self, X): probs = self.net.predict_proba(X) if not isfinite(probs).sum(): errmsg = 'network "{0:s}" predicted infinite/NaN probabilities'.format( self.name) stderr.write(errmsg) raise DivergenceError(errmsg) return probs def predict(self, X): return self.net.predict(X) def score(self, X, y, **kwargs): return self.net.score(X, y) def save(self, filepath=None): assert hasattr( self, 'net' ), 'Cannot save a network that is not initialized; .fit(X, y) something first [or use net.initialize(..) for random initialization].' parameters = self.get_params(deep=False) filepath = filepath or join(NNET_STATE_DIR, self.name) if VERBOSITY >= 1: print 'saving network to "{0:s}.net.npz|json"'.format(filepath) with open(filepath + '.net.json', 'w+') as fh: dump([parameters, self.feature_count, self.class_count], fp=fh, indent=2) save_knowledge(self.net, filepath + '.net.npz') @classmethod def load(cls, filepath=None, name=None): """ :param filepath: The base path (without extension) to load the file from, OR: :param name: The name of the network to load (if filename is not given) :return: The loaded network """ filepath = filepath or join(NNET_STATE_DIR, name) if VERBOSITY >= 1: print 'loading network from "{0:s}.net.npz|json"'.format(filepath) with open(filepath + '.net.json', 'r') as fh: [parameters, feature_count, class_count] = load(fp=fh) nnet = cls(**parameters) nnet.init_net(feature_count=feature_count, class_count=class_count) load_knowledge(nnet.net, filepath + '.net.npz') return nnet
class graspNet(): def __init__(self, param_file=None): net_divider = 1.0 self.layers = [ (L.layers.InputLayer, { 'shape': (None, 7, X_H, X_W), 'name': 'input' }), (L.layers.Conv2DLayer, { 'num_filters': 96, 'stride': 1, 'pad': 3, 'filter_size': (7, 7), 'nonlinearity': L.nonlinearities.rectify, 'flip_filters': False, 'name': 'conv0' }), (L.layers.DropoutLayer, { 'p': 0.0 }), (L.layers.MaxPool2DLayer, { 'pool_size': 3, 'ignore_border': False }), (L.layers.Conv2DLayer, { 'num_filters': 256, 'stride': 1, 'filter_size': (5, 5), 'nonlinearity': L.nonlinearities.rectify, 'flip_filters': False, 'name': 'conv1' }), (L.layers.DropoutLayer, { 'p': 0.0 }), (L.layers.MaxPool2DLayer, { 'pool_size': 2, 'ignore_border': False }), (L.layers.Conv2DLayer, { 'num_filters': 512, 'stride': 1, 'pad': 1, 'filter_size': (3, 3), 'nonlinearity': L.nonlinearities.rectify, 'flip_filters': False, 'name': 'conv2' }), (L.layers.DropoutLayer, { 'p': 0.0 }), (L.layers.Conv2DLayer, { 'num_filters': 512, 'stride': 1, 'pad': 1, 'filter_size': (3, 3), 'nonlinearity': L.nonlinearities.rectify, 'flip_filters': False, 'name': 'conv3' }), (L.layers.DropoutLayer, { 'p': 0.0 }), (L.layers.Conv2DLayer, { 'num_filters': 512, 'stride': 1, 'pad': 1, 'filter_size': (3, 3), 'nonlinearity': L.nonlinearities.rectify, 'flip_filters': False, 'name': 'conv4' }), (L.layers.DropoutLayer, { 'p': 0.0 }), (L.layers.MaxPool2DLayer, { 'pool_size': 3, 'ignore_border': False }), (L.layers.DenseLayer, { 'num_units': 4096, 'nonlinearity': L.nonlinearities.sigmoid, 'name': 'dense0' }), (L.layers.DropoutLayer, { 'p': 0.0 }), (L.layers.DenseLayer, { 'num_units': 4096, 'nonlinearity': L.nonlinearities.sigmoid, 'name': 'dense1' }), (L.layers.DenseLayer, { 'num_units': 1, 'nonlinearity': L.nonlinearities.sigmoid }), ] self.net = NeuralNet( layers=self.layers, update_learning_rate=0.015, update=L.updates.nesterov_momentum, update_momentum=0.9, #update=L.updates.sgd, regression=True, verbose=1, eval_size=0.15, objective_loss_function=L.objectives.binary_crossentropy, max_epochs=200) if param_file != None: print "Loading parameters from ", param_file self.net.load_params_from(param_file) def write_filters_to_file(self, fname): params = self.net.get_all_params() layer_counter = 0 for p in params: print p layer_counter += 1 filter_counter = 0 weights = p.get_value() if len(weights.shape) > 2: for f in weights: kernel = np.asarray(f, dtype=np.float32) kernel = kernel * 255 + 128 viz = np.zeros(shape=kernel[0].shape) viz = cv2.resize(kernel[0], None, fx=20, fy=20, interpolation=cv2.INTER_CUBIC) cv2.normalize(viz, viz) viz = viz * 50 #print viz #cv2.imshow('ha', viz) #cv2.waitKey(-1) viz = viz / 50 viz = viz * 12000 viz = viz #print viz cv2.imwrite( fname + "_" + str(layer_counter) + "_" + str(filter_counter) + ".png", viz) filter_counter += 1 # evaluates the input array over the neural net def eval(self, x): y = np.zeros((x.shape[0], )) for i in range(x.shape[0]): pred = self.net.predict(np.array([x[i]])) y[i] = pred print i, pred return y # train the network in input (x,y) # param_file: input parameter file (perhaps from previous trainings) # out_param_file: output path to write resulting params to # filter_file: output path to write images of filters for visualization # load_params: boolean flag, when set: 1 loads input parameters from <param_file> # pretrain: boolean flag, when set: 1 loads parameters from pretrained network def train(self, X, y, param_file=None, out_param_file=None, filter_file=None, load_params=False, pretrain=False): if pretrain: params = createNoLearnParams(param_file) print "Parameters", params[1].shape conv0_W = np.concatenate((params[0], params[0]), axis=1) conv0_W = np.concatenate((conv0_W, params[0]), axis=1) conv0_W = conv0_W[:, :7, :, :] conv0_b = np.concatenate((params[1], params[1]), axis=0) conv0_b = np.concatenate((conv0_b, params[1]), axis=0) conv0_b = conv0_b[:96] conv1_W = np.concatenate((params[2], params[2]), axis=1) conv1_W = np.concatenate((conv1_W, params[2]), axis=1) conv1_W = conv1_W[:, :96, :, :] conv1_b = np.concatenate((params[3], params[3]), axis=0) conv1_b = np.concatenate((conv1_b, params[3]), axis=0) conv1_b = conv1_b[:256] conv2_W = np.concatenate((params[4], params[4]), axis=1) conv2_W = np.concatenate((conv2_W, params[4]), axis=1) conv2_W = conv2_W[:, :256, :, :] conv2_b = np.concatenate((params[5], params[5]), axis=0) conv2_b = np.concatenate((conv2_b, params[5]), axis=0) conv2_b = conv2_b[:512] conv3_W = np.concatenate((params[6], params[6]), axis=1) conv3_W = np.concatenate((conv3_W, params[6]), axis=1) conv3_W = conv3_W[:, :512, :, :] conv3_b = np.concatenate((params[7], params[7]), axis=0) conv3_b = np.concatenate((conv3_b, params[7]), axis=0) conv3_b = conv3_b[:512] conv4_W = np.concatenate((params[8], params[8]), axis=1) conv4_W = np.concatenate((conv4_W, params[8]), axis=1) conv4_W = conv4_W[:, :512, :, :] conv4_b = np.concatenate((params[9], params[9]), axis=0) conv4_b = np.concatenate((conv4_b, params[9]), axis=0) conv4_b = conv4_b[:512] dense0_W = np.concatenate((params[10], params[10]), axis=1) dense0_W = np.concatenate((dense0_W, params[10]), axis=1) dense0_W = dense0_W[:2560, :4096] dense0_b = np.concatenate((params[11], params[11]), axis=0) dense0_b = np.concatenate((dense0_b, params[11]), axis=0) dense0_b = dense0_b[:4096] dense1_W = np.concatenate((params[12], params[12]), axis=1) dense1_W = np.concatenate((dense1_W, params[12]), axis=1) dense1_W = dense1_W[:4096, :4096] dense1_b = np.concatenate((params[13], params[13]), axis=0) dense1_b = np.concatenate((dense1_b, params[13]), axis=0) dense1_b = dense1_b[:4096] #http://arxiv.org/pdf/1405.3531v4.pdf self.net = NeuralNet( layers=self.layers, conv0_W=np.array(conv0_W), conv0_b=np.array(conv0_b), conv1_W=np.array(conv1_W), conv1_b=np.array(conv1_b), conv2_W=np.array(conv2_W), conv2_b=np.array(conv2_b), conv3_W=np.array(conv3_W), conv3_b=np.array(conv3_b), conv4_W=np.array(conv4_W), conv4_b=np.array(conv4_b), dense0_W=np.array(dense0_W), dense0_b=np.array(dense0_b), dense1_W=np.array(dense1_W), dense1_b=np.array(dense1_b), update_learning_rate=0.015, update=L.updates.nesterov_momentum, update_momentum=0.9, #update=L.updates.sgd, regression=True, verbose=1, eval_size=0.15, objective_loss_function=L.objectives.binary_crossentropy, max_epochs=200) if load_params: print "Loading parameters from ", param_file self.net.load_params_from(param_file) print "TRAINING!" print "input shape: ", X.shape print "output shape: ", y.shape print "Example X", X[0] print "Example Y", y[0] #print self.net.get_params() self.net.fit(X, y) print(self.net.score(X, y)) print "Saving network parameters to ", out_param_file, "..." file = open(out_param_file, 'w+') file.close() self.net.save_weights_to(out_param_file) print "Saving filters to ", filter_file self.write_filters_to_file(filter_file) plt = visualize.plot_loss(self.net) plt.show() plt.savefig(DIR_PROJ + 'loss.png') plt.clf() plt.cla() print "Sample predictions" for i in range(10): pred = self.net.predict(np.array([X[i]])) print "---------------------------------" print i print "prediction", pred print y[i]
def build_nn(df=None, class_column_name=None): """ Construct a classification neural network model from input dataframe Parameters: df : input dataframe class_column_name : identity of the column in df with class data """ # Type check inputs for sanity if df is None: raise ValueError('df is None') if not isinstance(df, pd.DataFrame): raise TypeError('df is not a dataframe') if class_column_name is None: raise ValueError('class_column_name is None') if not isinstance(class_column_name, basestring): raise TypeError('class_column_name is not a string') if class_column_name not in df.columns: raise ValueError('class_column_name (%s) is not a valid column name' % class_column_name) df = df.sample(frac=1).reset_index(drop=True) df_train, df_test = train_test_split(df, TEST_SIZE) df_train, df_val = df_train[:(0.75 * len(df_train.index)), :], df_train[(0.75 * len(df_train.index)):, :] x_train, x_val, x_test = df_train, df_val, df_test # Remove the classification column from the dataframe x_train = x_train.drop(class_column_name, axis=1, inplace=True).values x_val = x_val.drop(class_column_name, axis=1, inplace=True).values x_test = x_test.drop(class_column_name, axis=1, inplace=True).values y_train = df_train[class_column_name].values.astype(np.int32) y_val = df_val[class_column_name].values.astype(np.int32) y_test = df_test[class_column_name].values.astype(np.int32) # Create classification model net = NeuralNet(layers=[('input', InputLayer), ('hidden0', DenseLayer), ('hidden1', DenseLayer), ('output', DenseLayer)], input_shape=(None, x_train.shape[1]), hidden0_num_units=NODES, hidden0_nonlinearity=nonlinearities.softmax, hidden1_num_units=NODES, hidden1_nonlinearity=nonlinearities.softmax, output_num_units=len(np.unique(y_train)), output_nonlinearity=nonlinearities.softmax, update_learning_rate=0.1, verbose=1, max_epochs=100) param_grid = {'hidden0_num_units': [4, 17, 25], 'hidden0_nonlinearity': [nonlinearities.sigmoid, nonlinearities.softmax], 'hidden1_num_units': [4, 17, 25], 'hidden1_nonlinearity': [nonlinearities.sigmoid, nonlinearities.softmax], 'update_learning_rate': [0.01, 0.1, 0.5]} grid_search = GridSearchCV(net, param_grid, verbose=0) grid_search.fit(x_train, y_train) net.fit(x_train, y_train) print(net.score(x_train, y_train)) with open(PICKLE, 'wb') as file: pickle.dump(x_train, file, pickle.HIGHEST_PROTOCOL) pickle.dump(y_train, file, pickle.HIGHEST_PROTOCOL) pickle.dump(df_test, file, pickle.HIGHEST_PROTOCOL) pickle.dump(grid_search, file, pickle.HIGHEST_PROTOCOL) pickle.dump(net, file, pickle.HIGHEST_PROTOCOL)
def build_net(train, test, y_scaler): xs_test, ys_test = test xs_train, ys_train = train num_features = xs_train.shape[1] #assert(num_features == len(feature_extractions().keys())) loss_function = get_loss_function(y_scaler) input_var = theano.tensor.dmatrix('inputs') target_var = theano.tensor.dvector('targets') # l_in = las.layers.InputLayer((len(xs_test), len(xs_test[0])), input_var=input_var) # l_recur_a = las.layers.RecurrentLayer(l_in, num_units= 50) # l_hidden = las.layers.DenseLayer(l_recur_a, num_units = 4,nonlinearity = las.nonlinearities.softmax, W=las.init.Normal(0.1)) # l_recur_b = las.layers.RecurrentLayer(l_hidden, num_units = 4) #Try doing custom # -----pure classes below c_l_in = las.layers.InputLayer c_l_recur_a = las.layers.RecurrentLayer c_l_hidden = las.layers.DenseLayer c_l_recur_b = las.layers.RecurrentLayer #Try doing custom c_expression_layer = las.layers.special.ExpressionLayer c_output = las.layers.DenseLayer #layers = [('input', c_l_in), ('a', c_l_recur_a), ('h', c_l_hidden), ('b', c_l_recur_b),('output', c_output)] layers = [('input', c_l_in), ('h', c_l_hidden), ('h2', c_l_hidden),('h3', c_l_hidden),('h4', c_l_hidden), ('output', c_output)] print "\nBuilding..." #o = binary_hinge_loss net0 = NeuralNet(layers=layers, regression=True, y_tensor_type=theano.tensor.type.TensorType('float64', (False, True)) , input_shape=(None, num_features), # input_input_var = input_var, # a_num_units = 50, h_num_units=400, #h_nonlinearity = las.nonlinearities.softmax, h2_num_units=50, h3_num_units=20, h4_num_units=1, # h2_nonlinearity = las.nonlinearities.softmax, # b_num_units = 4, #e_function=expression_layer_fn, output_num_units=1, # output_nonlinearity=softmax, objective_loss_function=loss_function, update=nesterov_momentum, update_learning_rate=0.001, update_momentum=0.3, train_split=nolearn.lasagne.TrainSplit(eval_size=0.1), verbose=1, max_epochs=1000) print "Begin training" net0.fit(xs_train, ys_train) print "y: %f" % (ys_test[0]) print "transformed y: %f" %(y_scaler.inverse_transform([ys_test[0]])[0]) print "\n" print "y: %f" % (ys_test[1]) print "transformed y: %f" % (y_scaler.inverse_transform([ys_test[1]])[0]) print "\n predictions: :" print "y: {}".format((net0.predict([xs_test[0], xs_test[1]]))) print y_scaler.inverse_transform(net0.predict([xs_test[0], xs_test[1]])) # predicts = net0.predict([[30.0,-1.5,4.5,3087],[1.0,1.0,1.0,1.0],[5.0,0.1,5.0,1000]]) # print "\nPrediction: %f - 93864 == %f \n %f - 3 == %f \n %f - 1000000 == %f" % (predicts[0], (predicts[0]-93864)*1.0/93864, predicts[1], (predicts[1] - 3)*1.0/3, predicts[2], (predicts[2]-1000000)/1000000) # print "\n\nTransformed:" # predicts = map(lambda x: y_scaler.inverse_transform([x]), predicts) # predicts = [y[0] for y in predicts] # print "prediction: %f - 93864 == %f \n %f - 3 == %f \n %f - 1000000 == %f" % (predicts[0], (predicts[0]-93864)*1.0/93864, predicts[1], (predicts[1] - 3)*1.0/3, predicts[2], (predicts[2]-1000000)/1000000) print "\n Scores:" print "test score: %f" % (net0.score(xs_test,ys_test)) print net0.score(xs_train,ys_train) print "random score: %f" % (net0.score(xs_test,ys_train[0:len(xs_test)])) print "random score: %f" % (net0.score(xs_train[0:len(ys_test)],ys_test)) print net0.layers
(DenseLayer, dict(name='out', num_units=n_classes, nonlinearity=nonlinearities.softmax)), ], regression=False, objective_loss_function=objectives.categorical_crossentropy, update=updates.adam, batch_iterator_train=train_iterator, batch_iterator_test=test_iterator, on_epoch_finished=[ save_weights, save_training_history, plot_training_history ], verbose=10, max_epochs=250, ) if __name__ == '__main__': # X_train, X_test are image file names # They will be read in the iterator X_train, X_test, y_train, y_test = load_data(test_size=0.25, random_state=42) net.fit(X_train, y_train) # Load the best weights from pickled model net.load_params_from('./examples/cifar10/model_weights.pkl') score = net.score(X_test, y_test) print 'Final score %.4f' % score
NN0 = NeuralNet(layers = layer0, max_epochs = 10, # optimization method: update=adam, update_learning_rate=0.0002 ) # In[159]: NN0.fit(x_train, y_train) # In[160]: NN0.score(x_vali, y_vali) # the accuracy is only 0.95, lower than the score of random forest of 0.966 # # model 1: one input layer, two hidden layers, and one output layer # In[172]: layer1=[(layers.InputLayer, {'shape': (None, 1, 28, 28)}), (layers.DenseLayer, {'num_units':1000}), (layers.DropoutLayer, {}), (layers.DenseLayer, {'num_units':1000}), (layers.DenseLayer, {'num_units':10, 'nonlinearity': softmax})] # In[173]:
def smart_find(X, y, X_valid, y_valid): loss = [] kf = KFold(n_splits=5, shuffle=True) conf_set = set() step = (64 + 10) / 4 max_neuron_units = step * 8 for i in range(1, max_neuron_units, step): for j in range(0, max_neuron_units, step): for k in range(0, max_neuron_units, step): struct_net = (i) l = InputLayer(shape=(None, X.shape[1])) # ------- HIDDEN ----------- l = DenseLayer(l, num_units=i, nonlinearity=softmax) if j > 0: if i + step < j: continue l = DenseLayer(l, num_units=j, nonlinearity=softmax) struct_net = (i, j) if k > 0: if i + step < k or j + step < k: continue struct_net = (i, j, k) l = DenseLayer(l, num_units=k, nonlinearity=softmax) # ------- HIDDEN ----------- l = DenseLayer(l, num_units=len(np.unique(y)), nonlinearity=softmax) net = NeuralNet(l, update=adam, update_learning_rate=0.01, max_epochs=250) if struct_net in conf_set: continue print('=' * 40) print(struct_net) print('=' * 40) conf_set.add(struct_net) k_loss = [] y_data = np.array([y]).transpose() data = np.concatenate((X, y_data), axis=1) for train_index, test_index in kf.split(data): X_train, X_test = X[train_index], X[test_index] y_train, y_test = y[train_index], y[test_index] net.fit(X_train, y_train) y_pred = net.predict(X_test) loss_error = net.score(X_test, y_test) # loss_error = mean_squared_error(y_test, y_pred) k_loss.append(loss_error) print(loss_error) loss_net = (i, j, k, np.array(k_loss).mean()) print(loss_net) loss.append(loss_net) print('=' * 40) # for i in range(1, max_hidden_layers): # for j in range((64 + 10) // 2 // i, max_neuron_units // i, 10 // i): # print('=' * 40) # print('%s hidden layers' % i) # print('%s neurons' % j) # print('=' * 40) # l = InputLayer(shape=(None, X.shape[1])) # for k in range(i): # l = DenseLayer(l, num_units=j, nonlinearity=softmax) # l = DenseLayer(l, num_units=len(np.unique(y)), nonlinearity=softmax) # net = NeuralNet(l, update=adam, update_learning_rate=0.01, max_epochs=500) # # k_loss = [] # y_data = np.array([y]).transpose() # data = np.concatenate((X, y_data), axis=1) # for train_index, test_index in kf.split(data): # X_train, X_test = X[train_index], X[test_index] # y_train, y_test = y[train_index], y[test_index] # # net.fit(X_train, y_train) # y_pred = net.predict(X_test) # loss_error = mean_squared_error(y_test, y_pred) # k_loss.append(loss_error) # print(loss_error) # # loss_net = (i, j, np.array(k_loss).mean()) # print(loss_net) # loss.append(loss_net) # print('=' * 40) print(min(loss, key=lambda x: x[3])) print(max(loss, key=lambda x: x[3])) print(loss)
print "X_training shape must match y_training shape" print "Generate X_test and y_test" n_input = 11 print "X_test..." print "Multi Layer Perceptron..." #Build layer for MLP l_in = ls.layers.InputLayer(shape=(None,10),input_var=None) l_hidden = ls.layers.DenseLayer(l_in,num_units=15,nonlinearity=ls.nonlinearities.sigmoid) network = l_out = ls.layers.DenseLayer(l_hidden,num_units=1) print "Neural network initialize" #Init Neural net net1 = NeuralNet( layers=network, # optimization method: update=nesterov_momentum, update_learning_rate=0.001, update_momentum=0.9, regression=True, # flag to indicate we're dealing with regression problem max_epochs=400, # we want to train this many epochs verbose=1, ) # print "Training time!!!!!....." net1.fit(X_training,y_training) net1.save_params_to("saveNeuralNetwork.tdn") print "Score rate = " print net1.score(n_sample2,n_test2) print net1.predict(n_sample2)[0:2]
(FeaturePoolLayer, dict(name='l8p', pool_size=2)), (DropoutLayer, dict(name='l8drop', p=0.5)), (DenseLayer, dict(name='out', num_units=10, nonlinearity=nonlinearities.softmax)), ], regression=False, objective_loss_function=objectives.categorical_crossentropy, update=updates.adam, update_learning_rate=1e-3, batch_iterator_train=train_iterator, batch_iterator_test=test_iterator, on_epoch_finished=[ save_training_history, plot_training_history, ], verbose=10, max_epochs=20 ) if __name__ == '__main__': X_train, X_test, y_train, y_test = load_data(test_size=0.25, random_state=42) print "Training Network" net.fit(X_train, y_train) score = net.score(X_test, y_test) print 'Final score %.4f' % score