def train(self, X, y_train, X_test, ids_test, y_test, outfile, is_valid): X = np.array(X) encoder = LabelEncoder() y = encoder.fit_transform(y_train).astype(np.int32) num_classes = len(encoder.classes_) num_features = X.shape[1] layers0 = [('input', InputLayer), ('dense1', DenseLayer), ('dropout1', DropoutLayer), ('dense2', DenseLayer), ('dropout2', DropoutLayer), ('output', DenseLayer)] net0 = NeuralNet(layers=layers0, input_shape=(None, num_features), dense1_num_units=3500, dropout1_p=0.4, dense2_num_units=2300, dropout2_p=0.5, output_num_units=num_classes, output_nonlinearity=softmax, #update=nesterov_momentum, update=adagrad, update_learning_rate=0.01, #update_momentum=0.9, #objective_loss_function=softmax, objective_loss_function=categorical_crossentropy, eval_size=0.2, verbose=1, max_epochs=20) net0.fit(X, y) X_test = np.array(X_test) self.make_submission(net0, X_test, ids_test, encoder)
def loadNet(netName): if os.path.exists(netName): net = pickle.load(open(netName, "rb")) else: net = NeuralNet( layers=[ # three layers: one hidden layer ('input', layers.InputLayer), ('hidden', layers.DenseLayer), ('output', layers.DenseLayer), ], # layer parameters: input_shape=(None, 9216), # 96x96 input pixels per batch hidden_num_units=100, # number of units in hidden layer output_nonlinearity=None, # output layer uses identity function output_num_units=30, # 30 target values # optimization method: update=nesterov_momentum, update_learning_rate=0.01, update_momentum=0.9, regression=True, # flag to indicate we're dealing with regression problem max_epochs=400, # we want to train this many epochs verbose=1, ) X, y = load() net.fit(X, y) print("X.shape == {}; X.min == {:.3f}; X.max == {:.3f}".format(X.shape, X.min(), X.max())) print("y.shape == {}; y.min == {:.3f}; y.max == {:.3f}".format(y.shape, y.min(), y.max())) pickle.dump(net, open(netName, 'wb'), -1) return net
def fit_model(train_x, y, test_x): """Feed forward neural network for kaggle digit recognizer competition. Intentionally limit network size and optimization time (by choosing max_epochs = 15) to meet runtime restrictions """ print("\n\nRunning Convetional Net. Optimization progress below\n\n") net1 = NeuralNet( layers=[ #list the layers here ('input', layers.InputLayer), ('hidden1', layers.DenseLayer), ('output', layers.DenseLayer), ], # layer parameters: input_shape=(None, train_x.shape[1]), hidden1_num_units=200, hidden1_nonlinearity=rectify, #params of first layer output_nonlinearity=softmax, # softmax for classification problems output_num_units=10, # 10 target values # optimization method: update=nesterov_momentum, update_learning_rate=0.05, update_momentum=0.7, regression=False, max_epochs=10, # Intentionally limited for execution speed verbose=1, ) net1.fit(train_x, y) predictions = net1.predict(test_x) return(predictions)
def lasagne_oneLayer_classifier(param, X, labels): ## initialize the NN layers0 = [('input', InputLayer), ('dense0', DenseLayer), ('dropout', DropoutLayer), ('output', DenseLayer)] net0 = NeuralNet(layers=layers0, input_shape=(None, param['num_features']), dense0_num_units=param['dense0_num_units'], dropout_p=param['dropout_p'], output_num_units=param['num_classes'], output_nonlinearity=softmax, update=nesterov_momentum, update_learning_rate=param['update_learning_rate'], update_momentum=param['update_momentum'], eval_size=0.02, verbose=1, max_epochs=param['max_epochs']) ## fit the results net0.fit(X, labels) return net0
def train(x_train, y_train): clf_nn = NeuralNet( layers=[ # three layers: one hidden layer ('input', layers.InputLayer), ('hidden1', layers.DenseLayer), ('hidden2', layers.DenseLayer), ('output', layers.DenseLayer), ], # layer parameters: input_shape=(None, 2538), # 784 input pixels per batch hidden1_num_units=100, # number of units in hidden layer hidden2_num_units=100, output_nonlinearity=nonlinearities.softmax, # output layer uses identity function output_num_units=10, # 10 target values # optimization method: update=nesterov_momentum, update_learning_rate=0.01, update_momentum=0.9, max_epochs=50, # we want to train this many epochs verbose=1, ) clf_nn.fit(x_train, y_train) return clf_nn
def train(): weather = load_weather() training = load_training() X = assemble_X(training, weather) print len(X[0]) mean, std = normalize(X) y = assemble_y(training) input_size = len(X[0]) learning_rate = theano.shared(np.float32(0.1)) net = NeuralNet( layers=[ ('input', InputLayer), ('hidden1', DenseLayer), ('dropout1', DropoutLayer), ('hidden2', DenseLayer), ('dropout2', DropoutLayer), ('output', DenseLayer), ], # layer parameters: input_shape=(None, input_size), hidden1_num_units=325, dropout1_p=0.4, hidden2_num_units=325, dropout2_p=0.4, output_nonlinearity=sigmoid, output_num_units=1, # optimization method: update=nesterov_momentum, update_learning_rate=learning_rate, update_momentum=0.9, # Decay the learning rate on_epoch_finished=[ AdjustVariable(learning_rate, target=0, half_life=1), ], # This is silly, but we don't want a stratified K-Fold here # To compensate we need to pass in the y_tensor_type and the loss. regression=True, y_tensor_type = T.imatrix, objective_loss_function = binary_crossentropy, max_epochs=85, eval_size=0.1, verbose=1, ) X, y = shuffle(X, y, random_state=123) net.fit(X, y) _, X_valid, _, y_valid = net.train_test_split(X, y, net.eval_size) probas = net.predict_proba(X_valid)[:,0] print("ROC score", metrics.roc_auc_score(y_valid, probas)) return net, mean, std
def OptNN(d1, h1, d2, h2, d3, start, stop, max_epochs): params2 = params.copy() on_epoch = [AdjustVariable('update_learning_rate', start = start, stop = stop), AdjustVariable('update_momentum', start = .9, stop = .999)] params2['dropout1_p'] = d1 params2['dropout2_p'] = d2 params2['dropout3_p'] = d3 params2['dropout4_p'] = d4 params2['hidden1_num_units'] = h1 params2['hidden2_num_units'] = h2 params2['hidden3_num_units'] = h3 params2['max_epochs'] = max_epochs params2['on_epoch_finished'] = on_epoch kcv = StratifiedKFold(Y, 5, shuffle = True) res = np.empty((len(Y), len(np.unique(Y)))); i = 1 CVScores = [] for train_idx, valid_idx in kcv: logger.info("Running fold %d...", i); i += 1 net = NeuralNet(**params2) net.set_params(eval_size = None) net.fit(X[train_idx], Y[train_idx]) res[valid_idx, :] = net.predict_proba(X[valid_idx]) CVScores.append(log_loss(Y[valid_idx], res[valid_idx])) return -np.mean(CVScores)
def trainNet(X, Y, ln, loadFile = ""): net1 = NeuralNet( layers=[ # four layers: two hidden layers ('input', layers.InputLayer), ('hidden', layers.DenseLayer), ('hidden1', layers.DenseLayer), ('output', layers.DenseLayer), ], # layer parameters: Best 400 400 input_shape=(None, numInputs), # 31 inputs hidden_num_units=400, # number of units in hidden layer hidden1_num_units=400, hidden_nonlinearity=lasagne.nonlinearities.sigmoid, hidden1_nonlinearity=lasagne.nonlinearities.sigmoid, output_nonlinearity=None, # output layer uses identity function output_num_units=numOutputs, # 4 outputs # optimization method: update=nesterov_momentum, update_learning_rate=ln, update_momentum=0.9, regression=True, # flag to indicate we're dealing with regression problem max_epochs=1500, # we want to train this many epochs verbose=1, ) #if (loadFile != ""): #net1.load_params_from(loadFile) net1.max_epochs = 10 net1.update_learning_rate = ln; net1.fit(X, Y) # This thing try to do the fit itself return net1
def fit(self,tr,add_feat_tr): ## if trend exists, remove trend if self.trend ==1: trend = self.est_trend(tr) tr = tr-np.asarray(trend) layers0=[ ## 2 layers with one hidden layer (InputLayer, {'shape': (None,8,self.window_length)}), (DenseLayer, {'num_units': 8*self.window_length}), (DropoutLayer, {'p':0.3}), (DenseLayer, {'num_units': 8*self.window_length/3}), ## the output layer (DenseLayer, {'num_units': 1, 'nonlinearity': None}), ] feats = build_feat(tr, add_feat_tr, window_length=self.window_length) print feats.shape feat_target = get_target(tr,window_length=self.window_length) print feat_target.shape net0 = NeuralNet( layers=layers0, max_epochs=400, update=nesterov_momentum, update_learning_rate=0.01, update_momentum=0.9, verbose=1, regression=True, ) net0.fit(feats[:-1],feat_target) return net0,feats,feat_target
def neural_network(x_train, y_train): X, y, encoder, scaler = load_train_data(x_train, y_train) num_classes = len(encoder.classes_) num_features = X.shape[1] layers0 = [ ("input", InputLayer), ("dropoutf", DropoutLayer), ("dense0", DenseLayer), ("dropout", DropoutLayer), ("dense1", DenseLayer), ("dropout2", DropoutLayer), ("output", DenseLayer), ] net0 = NeuralNet( layers=layers0, input_shape=(None, num_features), dropoutf_p=0.15, dense0_num_units=1000, dropout_p=0.25, dense1_num_units=500, dropout2_p=0.25, output_num_units=num_classes, output_nonlinearity=softmax, update=adagrad, update_learning_rate=0.005, eval_size=0.01, verbose=1, max_epochs=30, ) net0.fit(X, y) return (net0, scaler)
def fit(xTrain, yTrain, dense0_num=800, dropout_p=0.5, dense1_num=500, update_learning_rate=0.01, update_momentum=0.9, test_ratio=0.2, max_epochs=20): #update_momentum=0.9, test_ratio=0.2, max_epochs=20, train_fname='train.csv'): #xTrain, yTrain, encoder, scaler = load_train_data(train_fname) #xTest, ids = load_test_data('test.csv', scaler) num_features = len(xTrain[0,:]) num_classes = 9 print num_features layers0 = [('input', InputLayer), ('dense0', DenseLayer), ('dropout', DropoutLayer), ('dense1', DenseLayer), ('output', DenseLayer)] clf = NeuralNet(layers=layers0, input_shape=(None, num_features), dense0_num_units=dense0_num, dropout_p=dropout_p, dense1_num_units=dense1_num, output_num_units=num_classes, output_nonlinearity=softmax, update=nesterov_momentum, update_learning_rate=update_learning_rate, update_momentum=update_momentum, eval_size=test_ratio, verbose=1, max_epochs=max_epochs) clf.fit(xTrain, yTrain) ll_train = metrics.log_loss(yTrain, clf.predict_proba(xTrain)) print ll_train return clf
def NN(X,y): net1 = NeuralNet( layers=[ # three layers: one hidden layer ('input', layers.InputLayer), ('hidden', layers.DenseLayer), ('output', layers.DenseLayer), ], # layer parameters: input_shape=(None, 9216), # 96x96 input pixels per batch hidden_num_units=100, # number of units in hidden layer output_nonlinearity=None, # output layer uses identity function output_num_units=30, # 30 target values # optimization method: update=nesterov_momentum, update_learning_rate=0.01, update_momentum=0.9, regression=True, # flag to indicate we're dealing with regression problem max_epochs=400, # we want to train this many epochs verbose=1, ) net1.fit(X, y)
def train_net(X, y): net2 = NeuralNet( layers=[ ('input', layers.InputLayer), ('ncaa', NCAALayer), ('dropout1', layers.DropoutLayer), ('hidden', layers.DenseLayer), ('dropout2', layers.DropoutLayer), ('output', layers.DenseLayer), ], input_shape = (None, num_features * 2), ncaa_num_units = 128, dropout1_p=0.2, hidden_num_units=128, dropout2_p=0.3, output_nonlinearity=nonlinearities.sigmoid, output_num_units=1, update=nesterov_momentum, update_learning_rate=theano.shared(float32(0.01)), update_momentum=theano.shared(float32(0.9)), regression=True, # flag to indicate we're dealing with regression problem max_epochs=20, # we want to train this many epochs verbose=1, ) net2.fit(X, y) return net2
def nn_example(data): net1 = NeuralNet( layers=[('input', layers.InputLayer), ('hidden', layers.DenseLayer), ('output', layers.DenseLayer), ], # layer parameters: input_shape=(None, 28*28), hidden_num_units=100, # number of units in 'hidden' layer output_nonlinearity=lasagne.nonlinearities.softmax, output_num_units=10, # 10 target values for the digits 0, 1, 2, ..., 9 # optimization method: update=nesterov_momentum, update_learning_rate=0.01, update_momentum=0.9, max_epochs=10, verbose=1, ) # Train the network net1.fit(data['X_train'], data['y_train']) # Try the network on new data print("Feature vector (100-110): %s" % data['X_test'][0][100:110]) print("Label: %s" % str(data['y_test'][0])) print("Predicted: %s" % str(net1.predict([data['X_test'][0]])))
def gridsearch_alpha(self,learning_rate,index,params=None): hidden_unit = ((index+1)*2)/3 self.l_in = ls.layers.InputLayer(shape=(None,n_input),input_var=None) self.l_hidden = ls.layers.DenseLayer(self.l_in,num_units=15,nonlinearity=ls.nonlinearities.rectify) self.network = l_out = ls.layers.DenseLayer(self.l_hidden,num_units=1) list_results = np.array([learning_rate.shape[0]],dtype=np.float64) for item in learning_rate: #Init Neural net net1 = NeuralNet( layers=self.network, # optimization method: update=nesterov_momentum, update_learning_rate=item, update_momentum=0.9, regression=True, # flag to indicate we're dealing with regression problem max_epochs=800, # we want to train this many epochs # verbose=1, eval_size = 0.4 ) net1.fit(self.X_training,self.y_training) self.pred = net1.predict(self.n_sample2) name_file = "Params/saveNeuralNetwork_%s_%s.tdn" %(item,index) net1.save_params_to(name_file) score_nn = net1.score(self.n_sample2,self.n_test2) list_results[item] = score_nn print "index=%f,item=%f,score=%f"%(index,item,score_nn) return list_results
def build_mlp(input_var=None): net1 = NeuralNet( layers=[ # three layers: one hidden layer ('input', layers.InputLayer), ('hidden1', layers.DenseLayer), ('hidden2', layers.DenseLayer), ('output', layers.DenseLayer), ], # layer parameters: input_shape=(None, 14, 2177), # 14 x 2177 input pixels per batch hidden1_num_units=100, # number of units in hidden layer hidden2_num_units=100, output_nonlinearity=lasagne.nonlinearities.softmax, # output layer uses identity function output_num_units=2, # 2 target values # optimization method: update=nesterov_momentum, update_learning_rate=0.01, update_momentum=0.9, #regression=False, # flag to indicate we're dealing with regression problem max_epochs=500, # we want to train this many epochs verbose=1, ) X, y = load_dataset() y = np.asanyarray(y,np.int32) print(X.shape) print(y.shape) net1.fit(X, y)
def train_network(): layers0 = [('input', InputLayer), ('dense0', DenseLayer), ('dropout0', DropoutLayer), ('dense1', DenseLayer), ('dropout1', DropoutLayer), ('dense2', DenseLayer), ('output', DenseLayer)] es = EarlyStopping(patience=200) net0 = NeuralNet(layers=layers0, input_shape=(None, num_features), dense0_num_units=256, dropout0_p=0.5, dense1_num_units=128, dropout1_p=0.5, dense2_num_units=64, output_num_units=num_classes, output_nonlinearity=softmax, update=nesterov_momentum, update_learning_rate=theano.shared(float32(0.01)), update_momentum=theano.shared(float32(0.9)), eval_size=0.2, verbose=1, max_epochs=1000, on_epoch_finished=[ AdjustVariable('update_learning_rate', start=0.01, stop=0.0001), AdjustVariable('update_momentum', start=0.9, stop=0.999), es ]) net0.fit(X, y) return (es.best_valid, net0)
class NN(object): def __init__(self, input_size, hidden_1_size, hidden_2_size=None): n_layers = [ ('input', layers.InputLayer), ('hidden1', layers.DenseLayer), ('dropout1', layers.DropoutLayer) ] if hidden_2_size is not None: n_layers.extend( [('hidden2', layers.DenseLayer), ('dropout2', layers.DropoutLayer)] ) n_layers.append(('output', layers.DenseLayer)) self.model = NeuralNet( layers=n_layers, input_shape=(None, input_size), hidden1_num_units=hidden_1_size, dropout1_p=0.5, output_nonlinearity=tanh, output_num_units=1, regression=True, update=nesterov_momentum, update_learning_rate=0.01, update_momentum=0.9, eval_size=0.1, on_epoch_finished=[ AdjustVariable('update_learning_rate', stop=0.0001, decrement=0.00001), AdjustVariable('update_momentum', stop=0.999, increment=0.0001), EarlyStopping(patience=100) ], max_epochs=5000, verbose=1 ) if hidden_2_size is not None: self.model.__dict__['hidden2_num_units'] = hidden_2_size self.model.__dict__['dropout2_p'] = 0.5 def train(self, X, Y): self.model.fit(np.asarray(X, dtype=np.float32), np.asarray(Y, dtype=np.float32)) def predict_continuous(self, X_test): return self.model.predict(np.asarray(X_test, dtype=np.float32)) def predict_classes(self, X_test): Y_pred = self.predict_continuous(X_test) # threshold the continuous values to get the classes pos = Y_pred >= .33 neg = Y_pred <= -0.33 neu = np.logical_and(Y_pred < 0.33, Y_pred > -0.33) Y_pred[pos] = 1 Y_pred[neg] = -1 Y_pred[neu] = 0 return Y_pred.reshape(-1)
def regr(X, Y): l = InputLayer(shape=(None, X.shape[1])) l = DenseLayer(l, num_units=X.shape[1], nonlinearity=tanh) #tanh, sigmoid # l = DropoutLayer(l, p=0.3, rescale=True) # previous: p=0.5 l = DenseLayer(l, num_units=1, nonlinearity=sigmoid) # l = DropoutLayer(l, p=0.3, rescale=True) # previous: p=0.5 net = NeuralNet(l, regression=True, update_learning_rate=0.01, verbose=1, max_epochs=700) net.fit(X, Y) print(net.score(X, Y)) return net
class RegressionNN(RegressionBase.RegressionBase): def __init__(self, isTrain, isNN): super(RegressionNN, self).__init__(isTrain, isNN) # data preprocessing #self.dataPreprocessing() self.net1 = NeuralNet( layers=[ # three layers: one hidden layer ('input', layers.InputLayer), ('hidden', layers.DenseLayer), #('hidden2', layers.DenseLayer), #('hidden3', layers.DenseLayer), ('output', layers.DenseLayer), ], # layer parameters: input_shape=(None, 13), # input dimension is 13 hidden_num_units=6, # number of units in hidden layer #hidden2_num_units=8, # number of units in hidden layer #hidden3_num_units=4, # number of units in hidden layer output_nonlinearity=None, # output layer uses sigmoid function output_num_units=1, # output dimension is 1 # obejctive function objective_loss_function = lasagne.objectives.squared_error, # optimization method: update=lasagne.updates.nesterov_momentum, update_learning_rate=0.002, update_momentum=0.4, # use 25% as validation train_split=TrainSplit(eval_size=0.2), regression=True, # flag to indicate we're dealing with regression problem max_epochs=100, # we want to train this many epochs verbose=0, ) def dataPreprocessing(self): # due to the observation, standization does not help the optimization. # So do not use it! #self.Standardization() pass def training(self): # train the NN model self.net1.fit(self.X_train, self.y_train) def predict(self): # predict the test data self.y_pred = self.net1.predict(self.X_test) # print MSE mse = mean_squared_error(self.y_pred, self.y_test) print "MSE: {}".format(mse)
def regr(X, Y): l = InputLayer(shape=(None, X.shape[1])) l = DenseLayer(l, num_units=Y.shape[1]+100, nonlinearity=tanh) # l = DropoutLayer(l, p=0.3, rescale=True) # previous: p=0.5 l = DenseLayer(l, num_units=Y.shape[1]+50, nonlinearity=tanh) # l = DropoutLayer(l, p=0.3, rescale=True) # previous: p=0.5 l = DenseLayer(l, num_units=Y.shape[1], nonlinearity=None) net = NeuralNet(l, regression=True, update_learning_rate=0.1, verbose=1) net.fit(X, Y) print(net.score(X, Y)) return net
def OptNN2(d0, d1,d2, d3, h1, h2, h3, me, ls, le): h1, h2, h3 = int(h1), int(h2), int(h3); me = int(me) params = dict( layers = [ ('input', layers.InputLayer), ('dropout1', layers.DropoutLayer), ('hidden1', layers.DenseLayer), ('dropout2', layers.DropoutLayer), ('hidden2', layers.DenseLayer), ('dropout3', layers.DropoutLayer), ('hidden3', layers.DenseLayer), ('dropout4', layers.DropoutLayer), ('output', layers.DenseLayer), ], input_shape = (None, 93), dropout1_p = d0, hidden1_num_units = h1, dropout2_p = d1, hidden2_num_units = h2, dropout3_p = d2, hidden3_num_units = h3, dropout4_p = d3, output_nonlinearity = softmax, output_num_units = 9, update = nesterov_momentum, update_learning_rate = theano.shared(float32(l_start)), update_momentum = theano.shared(float32(m_start)), regression = False, on_epoch_finished = [ AdjustVariable('update_learning_rate', start = ls, stop = le, is_log = True), AdjustVariable('update_momentum', start = m_start, stop = m_stop, is_log = False), ], max_epochs = me, verbose = 1, ) CVScores = [] res = np.empty((len(Y), len(np.unique(Y)))) kcv = StratifiedKFold(Y, 5, shuffle = True); i = 1 for train_idx, valid_idx in kcv: logger.info("Running fold %d...", i); i += 1 net = NeuralNet(**params) net.set_params(eval_size = None) net.fit(X[train_idx], Y[train_idx]) res[valid_idx, :] = net.predict_proba(X[valid_idx]) CVScores.append(log_loss(Y[valid_idx], res[valid_idx])) return -np.mean(CVScores)
def test_lasagne_functional_regression(boston): from nolearn.lasagne import NeuralNet X, y = boston layer1 = InputLayer(shape=(128, 13)) layer2 = DenseLayer(layer1, num_units=100) output = DenseLayer(layer2, num_units=1, nonlinearity=identity) nn = NeuralNet(layers=output, update_learning_rate=0.01, update_momentum=0.1, regression=True, max_epochs=50) nn.fit(X[:300], y[:300]) assert mean_absolute_error(nn.predict(X[300:]), y[300:]) < 3.0
def linreg(X, y): net = NeuralNet( layers=[ ('input', InputLayer), ('output', DenseLayer)], input_shape=(None, X.shape[1]), output_num_units=2, output_nonlinearity=identity, regression=True, update_learning_rate=1e-9, update_momentum=0.9, verbose=1) net.fit(X, y)
def test_lasagne_functional_mnist(mnist): # Run a full example on the mnist dataset from nolearn.lasagne import NeuralNet X, y = mnist X_train, y_train = X[:60000], y[:60000] X_test, y_test = X[60000:], y[60000:] epochs = [] def on_epoch_finished(nn, train_history): epochs[:] = train_history if len(epochs) > 1: raise StopIteration() nn = NeuralNet( layers=[ ('input', InputLayer), ('hidden1', DenseLayer), ('dropout1', DropoutLayer), ('hidden2', DenseLayer), ('dropout2', DropoutLayer), ('output', DenseLayer), ], input_shape=(None, 784), output_num_units=10, output_nonlinearity=softmax, more_params=dict( hidden1_num_units=512, hidden2_num_units=512, ), update=nesterov_momentum, update_learning_rate=0.01, update_momentum=0.9, max_epochs=5, on_epoch_finished=on_epoch_finished, ) nn.fit(X_train, y_train) assert len(epochs) == 2 assert epochs[0]['valid_accuracy'] > 0.85 assert epochs[1]['valid_accuracy'] > epochs[0]['valid_accuracy'] assert sorted(epochs[0].keys()) == [ 'epoch', 'train_loss', 'valid_accuracy', 'valid_loss', ] y_pred = nn.predict(X_test) assert accuracy_score(y_pred, y_test) > 0.85
class network(object): def __init__(self,X_train, Y_train): #self.__hidden=0 self.__hidden=int(math.ceil((2*(X_train.shape[1]+ 1))/3)) self.net= NeuralNet( layers=[ ('input', layers.InputLayer), ('hidden', layers.DenseLayer), ('output', layers.DenseLayer) ], input_shape=( None, X_train.shape[1] ), hidden_num_units=self.__hidden, #hidden_nonlinearity=nonlinearities.tanh, output_nonlinearity=None, batch_iterator_train=BatchIterator(batch_size=256), output_num_units=1, on_epoch_finished=[EarlyStopping(patience=50)], update=momentum, update_learning_rate=theano.shared(np.float32(0.03)), update_momentum=theano.shared(np.float32(0.8)), regression=True, max_epochs=1000, verbose=1, ) self.net.fit(X_train,Y_train) def predict(self,X): return self.net.predict(X) def showMetrics(self): train_loss = np.array([i["train_loss"] for i in self.net.train_history_]) valid_loss = np.array([i["valid_loss"] for i in self.net.train_history_]) pyplot.plot(train_loss, linewidth=3, label="training") pyplot.plot(valid_loss, linewidth=3, label="validation") pyplot.grid() pyplot.legend() pyplot.xlabel("epoch") pyplot.ylabel("loss") # pyplot.ylim(1e-3, 1e-2) pyplot.yscale("log") pyplot.show() def saveNet(self,fname): self.net.save_params_to(fname) def loadNet(self,fname): self.net.load_params_from(fname)
def nnet(pipe): pipe.features = pipe.features.astype(np.float32) pipe.labels = pipe.labels.astype(np.int32) pipe.features = StandardScaler().fit_transform(pipe.features) X_train, X_test, y_train, y_test = train_test_split(pipe.features, pipe.labels) nnet = NeuralNet( # Specify the layers layers=[('input', layers.InputLayer), ('hidden1', layers.DenseLayer), ('hidden2', layers.DenseLayer), ('hidden3', layers.DenseLayer), ('output', layers.DenseLayer)], # Input Layer input_shape=(None, pipe.features.shape[1]), # Hidden Layer 1 hidden1_num_units=512, hidden1_nonlinearity=rectify, # Hidden Layer 2 hidden2_num_units=512, hidden2_nonlinearity=rectify, # # Hidden Layer 3 hidden3_num_units=512, hidden3_nonlinearity=rectify, # Output Layer output_num_units=2, output_nonlinearity=softmax, # Optimization update=nesterov_momentum, update_learning_rate=0.001, update_momentum=0.3, max_epochs=30, # Others, regression=False, verbose=1, ) nnet.fit(X_train, y_train) y_predict = nnet.predict(X_test) print "precision for nnet:", precision_score(y_test, y_predict) print "recall for nnet:", recall_score(y_test, y_predict) print "f1 for nnet:", f1_score(y_test, y_predict, average='weighted') pickle.dump( nnet, open( "model.pkl", "wb" ), protocol = cPickle.HIGHEST_PROTOCOL)
def mlp(X, y): net = NeuralNet( layers=[ ('input', InputLayer), ('hidden', DenseLayer), ('output', DenseLayer)], input_shape=(None, X.shape[1]), hidden_num_units=5, hidden_nonlinearity=sigmoid, output_num_units=2, output_nonlinearity=sigmoid, regression=True, update_learning_rate=1e-3, update_momentum=0.9, verbose=1) net.fit(X, y)
def classif_no_valid(self, NeuralNet, X, y): from nolearn.lasagne import TrainSplit l = InputLayer(shape=(None, X.shape[1])) l = DenseLayer(l, num_units=len(np.unique(y)), nonlinearity=softmax) net = NeuralNet( l, update_learning_rate=0.01, train_split=TrainSplit(0)) return net.fit(X, y)
def net_fitted(self, NeuralNet, X_train, y_train): nn = NeuralNet( layers=[ ('input', InputLayer), ('conv1', Conv2DLayer), ('conv2', Conv2DLayer), ('pool2', MaxPool2DLayer), ('output', DenseLayer), ], input_shape=(None, 1, 28, 28), output_num_units=10, output_nonlinearity=softmax, more_params=dict( conv1_filter_size=(5, 5), conv1_num_filters=16, conv2_filter_size=(3, 3), conv2_num_filters=16, pool2_pool_size=(8, 8), hidden1_num_units=16, ), update=nesterov_momentum, update_learning_rate=0.01, update_momentum=0.9, max_epochs=3, ) return nn.fit(X_train, y_train)
input_shape=(None, 1, PIXELS, PIXELS), conv1_num_filters=32, conv1_filter_size=(3, 3), pool1_ds=(2, 2), conv2_num_filters=64, conv2_filter_size=(2, 2), pool2_ds=(2, 2), hidden4_num_units=500, output_num_units=2, output_nonlinearity=nonlinearities.softmax, update_learning_rate=0.01, update_momentum=0.9, regression=False, max_epochs=1000, verbose=1, ) class SimpleBatchIterator(BatchIterator): def transform(self, Xb, yb): Xb, yb = super(SimpleBatchIterator, self).transform(Xb, yb) # The 'incomming' and outcomming shape is (batchsize, 1, 28, 28) return Xb[:,:,::-1,:], yb #<--- Here we do the flipping net1.batch_iterator_train = SimpleBatchIterator(batch_size=128) net1.fit(X, y)
hidden3_nonlinearity=rectify, # Output Layer output_num_units=3, output_nonlinearity=softmax, # Optimization update=nesterov_momentum, update_learning_rate=0.001, update_momentum=0.3, max_epochs=30, # Others, regression=False, verbose=1, ) # # Train the NN print 'Training Neural Network' nnet.fit(X_train, y_train) print # Make predictions y_predictions = nnet.predict(X_test) print "f1 score:", f1_score(y_test, y_predictions, average='weighted') print classification_report(y_test, y_predictions) with open(r"../Pickled-Models/three-artist-neural-network.pickle", "wb") as output_file: cPickle.dump(nnet, output_file, protocol=cPickle.HIGHEST_PROTOCOL)
dropout2_p=0.5, # output output_nonlinearity=lasagne.nonlinearities.softmax, output_num_units=10, # optimization method params update=nesterov_momentum, update_learning_rate=0.01, update_momentum=0.9, max_epochs=10, verbose=1, ) # In[17]: nn1 = net1.fit(X_train1, y_train) # In[22]: X_train_rescaling=X_train1/255 # In[23]: net2 = NeuralNet( layers=[('input', layers.InputLayer), ('conv2d1', layers.Conv2DLayer), ('maxpool1', layers.MaxPool2DLayer), ('conv2d2', layers.Conv2DLayer), ('maxpool2', layers.MaxPool2DLayer),
class RegressionUniformBlending(RegressionBase.RegressionBase): def __init__(self, isTrain): super(RegressionUniformBlending, self).__init__(isTrain) # data preprocessing #self.dataPreprocessing() self.net1 = NeuralNet( layers=[ # three layers: one hidden layer ('input', layers.InputLayer), ('hidden', layers.DenseLayer), #('hidden2', layers.DenseLayer), #('hidden3', layers.DenseLayer), ('output', layers.DenseLayer), ], # layer parameters: input_shape=(None, 13), # input dimension is 13 hidden_num_units=6, # number of units in hidden layer #hidden2_num_units=8, # number of units in hidden layer #hidden3_num_units=4, # number of units in hidden layer output_nonlinearity=None, # output layer uses sigmoid function output_num_units=1, # output dimension is 1 # obejctive function objective_loss_function = lasagne.objectives.squared_error, # optimization method: update=lasagne.updates.nesterov_momentum, update_learning_rate=0.002, update_momentum=0.4, # use 25% as validation train_split=TrainSplit(eval_size=0.2), regression=True, # flag to indicate we're dealing with regression problem max_epochs=100, # we want to train this many epochs verbose=0, ) # Create linear regression object self.linRegr = linear_model.LinearRegression() # Create KNN regression object self.knn = neighbors.KNeighborsRegressor(86, weights='distance') # Create Decision Tree regression object self.decisionTree = DecisionTreeRegressor(max_depth=7, max_features=None) # Create AdaBoost regression object decisionReg = DecisionTreeRegressor(max_depth=10) rng = np.random.RandomState(1) self.adaReg = AdaBoostRegressor(decisionReg, n_estimators=400, random_state=rng) # Create linear regression object self.model = RandomForestRegressor(max_features='sqrt', n_estimators=32, max_depth=39) def dataPreprocessing(self): # due to the observation, standization does not help the optimization. # So do not use it! #self.Standardization() pass def training(self): # train each regression model self.net1.fit(self.X_train, self.y_train) self.linRegr.fit(self.X_train, self.y_train) self.knn.fit(self.X_train, self.y_train) self.decisionTree.fit(self.X_train, self.y_train) self.adaReg.fit(self.X_train, self.y_train.reshape((self.y_train.shape[0], ))) def predict(self): # predict the test data y_pred1 = self.net1.predict(self.X_test) y_pred1 = y_pred1.reshape((y_pred1.shape[0], 1)) y_pred2 = self.linRegr.predict(self.X_test) y_pred2 = y_pred2.reshape((y_pred2.shape[0], 1)) y_pred3 = self.knn.predict(self.X_test) y_pred3 = y_pred3.reshape((y_pred3.shape[0], 1)) y_pred4 = self.decisionTree.predict(self.X_test) y_pred4 = y_pred4.reshape((y_pred4.shape[0], 1)) y_pred5 = self.adaReg.predict(self.X_test) y_pred5 = y_pred5.reshape((y_pred5.shape[0], 1)) self.y_pred = (y_pred1+y_pred2+y_pred3+y_pred4+y_pred5)/5 # print MSE mse = mean_squared_error(self.y_pred, self.y_test) print "MSE: {}".format(mse)
('dropout2', layers.DropoutLayer), ('output', layers.DenseLayer), ], input_shape=(None, 1, 28, 28), conv2d1_num_filters=32, conv2d1_filter_size=(5, 5), conv2d1_nonlinearity=lasagne.nonlinearities.rectify, conv2d1_W=lasagne.init.GlorotUniform(), maxpool1_pool_size=(2, 2), conv2d2_num_filters=32, conv2d2_filter_size=(5, 5), conv2d2_nonlinearity=lasagne.nonlinearities.rectify, maxpool2_pool_size=(2, 2), dropout1_p=0.5, dense_num_units=256, dense_nonlinearity=lasagne.nonlinearities.rectify, dropout2_p=0.5, output_nonlinearity=lasagne.nonlinearities.softmax, output_num_units=10, update=nesterov_momentum, update_learning_rate=0.01, update_momentum=0.9, max_epochs=10, verbose=1, ) nn = CNN.fit(X_train, y_train) prediction = CNN.predict(X_test) visualize.plot_conv_weights(CNN.layers_['conv2d1'])
def classif(self, NeuralNet, X, y): l = InputLayer(shape=(None, X.shape[1])) l = DenseLayer(l, num_units=len(np.unique(y)), nonlinearity=softmax) net = NeuralNet(l, update_learning_rate=0.01) return net.fit(X, y)
conv2d1_W=lasagne.init.GlorotUniform(), # layer maxpool1 maxpool1_pool_size=(2, 2), # layer conv2d2 conv2d2_num_filters=32, conv2d2_filter_size=(5, 5), conv2d2_nonlinearity=lasagne.nonlinearities.rectify, # layer maxpool2 maxpool2_pool_size=(2, 2), # dropout1 dropout1_p=0.5, # dense dense_num_units=256, dense_nonlinearity=lasagne.nonlinearities.rectify, # dropout2 dropout2_p=0.5, # output output_nonlinearity=lasagne.nonlinearities.softmax, output_num_units=10, # optimization method params update=nesterov_momentum, update_learning_rate=0.01, update_momentum=0.9, max_epochs=10, verbose=1, ) # Train the network nn = net1.fit(X_train, y_train) visualize.plot_conv_weights(net1.layers_['conv2d1'])
def regr(self, NeuralNet, X, y): l = InputLayer(shape=(None, X.shape[1])) l = DenseLayer(l, num_units=y.shape[1], nonlinearity=None) net = NeuralNet(l, regression=True, update_learning_rate=0.01) return net.fit(X, y)
y_tensor_type=T.tensor4, #optimization parameters: update=nesterov_momentum, update_learning_rate=0.05, update_momentum=0.9, regression=True, max_epochs=50, verbose=1, batch_iterator_train=BatchIterator(batch_size=25), on_epoch_finished=[ EarlyStopping(patience=20), ], train_split=TrainSplit(eval_size=0.5)) net2.fit(train_x, train_y) plot_loss(net2) plt.savefig("/home/sandeep/Desktop/Major/Results/plotloss.png") plot_conv_weights(net2.layers_[1], figsize=(4, 4)) plt.savefig("/home/sandeep/Desktop/Major/Results/convweights.png") #layer_info = PrintLayerInfo() #layer_info(net2) import cPickle as pickle with open('/home/sandeep/Desktop/Major/Results/net2.pickle', 'wb') as f: pickle.dump(net2, f, -1) y_pred2 = net2.predict(test_x) print "The accuracy of this network is: %0.2f" % (abs(y_pred2 - test_y)).mean()
# update=adagrad, # update_learning_rate=.1, update=nesterov_momentum, update_learning_rate=theano.shared(np.float32(0.01)), update_momentum=theano.shared(np.float32(0.9)), # on_epoch_finished=[ AdjustVariable('update_learning_rate', start=0.01, stop=0.00001), AdjustVariable('update_momentum', start=0.9, stop=0.999), ], regression= False, # flag to indicate we're dealing with regression problem max_epochs=500, # we want to train this many epochs verbose=1) clf.fit(x_train, y_train) # if 1: # y_pred = clf.predict_proba(x_test) # # filename = 'testdata_aug_d' # savefile = open(filename+'.pkl', 'wb') # cPickle.dump((x_test, y_pred, name1),savefile,-1) # savefile.close() if 1: y_pred = clf.predict(x_test) print "Accuracy:", zero_one_loss(y_test, y_pred) print "Classification report:" print classification_report(y_test, y_pred) print 'Confusion matrix:' print confusion_matrix(y_test, y_pred)
conv2d1_num_filters=20, #20 filters, same for each channel? conv2d1_filter_size=(5, 5), conv2d1_stride=(1, 1), conv2d1_pad=(2, 2), conv2d1_nonlinearity=lasagne.nonlinearities.rectify, #standard reLu maxpool1_pool_size=(2, 2), conv2d2_num_filters=20, conv2d2_filter_size=(5, 5), conv2d2_stride=(1, 1), conv2d2_pad=(2, 2), conv2d2_nonlinearity=lasagne.nonlinearities.rectify, maxpool2_pool_size=(2, 2), dense_num_units=1000, dense_nonlinearity=lasagne.nonlinearities.rectify, output_nonlinearity=lasagne.nonlinearities.softmax, output_num_units=10, update=nesterov_momentum, update_momentum=0.9, update_learning_rate=0.0001, max_epochs=100, verbose=True) x_train, y_train, x_test, y_test = load_data( os.path.expanduser('~/deep-learning/data/cifar-10-batches-py')) network = net.fit(x_train, y_train) predictions = network.predict(x_test) print classification_report(y_test, predictions) print accuracy_score(y_test, predictions)
def main(): data = load_av_letters('data/allData_mouthROIs.mat') # create the necessary variable mappings data_matrix = data['dataMatrix'] data_matrix_len = data_matrix.shape[0] targets_vec = data['targetsVec'] vid_len_vec = data['videoLengthVec'] iter_vec = data['iterVec'] indexes = create_split_index(data_matrix_len, vid_len_vec, iter_vec) # split the data train_data = data_matrix[indexes == True] train_targets = targets_vec[indexes == True] test_data = data_matrix[indexes == False] test_targets = targets_vec[indexes == False] idx = [i for i, elem in enumerate(test_targets) if elem == 20] print(train_data.shape) print(test_data.shape) print(sum([train_data.shape[0], test_data.shape[0]])) # resize the input data to 40 x 30 train_data_resized = resize_images(train_data).astype(np.float32) # normalize the inputs [0 - 1] train_data_resized = normalize_input(train_data_resized, centralize=True) test_data_resized = resize_images(test_data).astype(np.float32) test_data_resized = normalize_input(test_data_resized, centralize=True) dic = {} dic['trainDataResized'] = train_data_resized dic['testDataResized'] = test_data_resized """second experiment: overcomplete sigmoid encoder/decoder, squared loss""" encode_size = 2500 sigma = 0.5 # to get tied weights in the encoder/decoder, create this shared weightMatrix # 1200 x 2000 w1, layer1 = build_encoder_layers(1200, 2500, sigma) ae1 = NeuralNet(layers=layer1, max_epochs=50, objective_loss_function=squared_error, update=adadelta, regression=True, verbose=1) load = True save = False if load: print('[LOAD] layer 1...') ae1.load_params_from('layer1.dat') else: print('[TRAIN] layer 1...') ae1.fit(train_data_resized, train_data_resized) # save params if save: print('[SAVE] layer 1...') ae1.save_params_to('layer1.dat') train_encoded1 = ae1.get_output('encoder', train_data_resized) # 12293 x 2000 w2, layer2 = build_encoder_layers(2500, 1250) ae2 = NeuralNet(layers=layer2, max_epochs=50, objective_loss_function=squared_error, update=adadelta, regression=True, verbose=1) load2 = True if load2: print('[LOAD] layer 2...') ae2.load_params_from('layer2.dat') else: print('[TRAIN] layer 2...') ae2.fit(train_encoded1, train_encoded1) save2 = False if save2: print('[SAVE] layer 2...') ae2.save_params_to('layer2.dat') train_encoded2 = ae2.get_output('encoder', train_encoded1) # 12293 x 1250 w3, layer3 = build_encoder_layers(1250, 600) ae3 = NeuralNet(layers=layer3, max_epochs=100, objective_loss_function=squared_error, update=adadelta, regression=True, verbose=1) load3 = True if load3: print('[LOAD] layer 3...') ae3.load_params_from('layer3.dat') else: ae3.fit(train_encoded2, train_encoded2) save3 = False if save3: print('[SAVE] layer 3...') ae3.save_params_to('layer3.dat') train_encoded3 = ae3.get_output('encoder', train_encoded2) # 12293 x 1250 w4, layer4 = build_bottleneck_layer(600, 100) ae4 = NeuralNet(layers=layer4, max_epochs=100, objective_loss_function=squared_error, update=adadelta, regression=True, verbose=1) load4 = False if load4: print('[LOAD] layer 4...') ae4.load_params_from('layer4.dat') else: print('[TRAIN] layer 4...') ae4.fit(train_encoded3, train_encoded3) save4 = True if save4: print('[SAVE] layer 4...') ae4.save_params_to('layer4.dat') test_enc1 = ae1.get_output('encoder', test_data_resized) test_enc2 = ae2.get_output('encoder', test_enc1) test_enc3 = ae3.get_output('encoder', test_enc2) test_enc4 = ae4.get_output('encoder', test_enc3) decoder4 = create_decoder(100, 600, w4.T) decoder4.initialize() decoder3 = create_decoder(600, 1250, w3.T) decoder3.initialize() decoder2 = create_decoder(1250, 2500, w2.T) decoder2.initialize() # initialize the net decoder1 = create_decoder(2500, 1200, w1.T) decoder1.initialize() test_dec3 = decoder4.predict(test_enc4) test_dec2 = decoder3.predict(test_dec3) test_dec1 = decoder2.predict(test_dec2) X_pred = decoder1.predict(test_dec1) # plot_loss(ae3) # plot_loss(ae2) # plot_loss(ae1) tile_raster_images(X_pred[4625:4650, :], (30, 40), (5, 5), tile_spacing=(1, 1)) plt.title('reconstructed') tile_raster_images(test_data_resized[4625:4650, :], (30, 40), (5, 5), tile_spacing=(1, 1)) plt.title('original') plt.show() """
update_learning_rate=0.05, update_momentum=0.5, max_epochs=25, #update = adagrad, #update_learning_rate = .07, #max_epochs = 50, # Others regression=False, verbose=1, ) # In[221]: nn.fit(train_x, train_y) predict_y = nn.predict(test_x) print "Accuracy Score: " +str(accuracy_score(test_y, predict_y)) print "Precision Score: " + str(precision_score(test_y, predict_y)) print "Recall Score: " + str(recall_score(test_y, predict_y)) print "F1 Score: " + str(f1_score(test_y, predict_y)) # In[ ]:
max_epochs=200) net2 = NeuralNet(layers=[('input', layers.InputLayer), ('conv1', layers.Conv2DLayer), ('pool1', layers.MaxPool2DLayer), ('conv2', layers.Conv2DLayer), ('pool2', layers.MaxPool2DLayer), ('conv3', layers.Conv2DLayer), ('pool3', layers.MaxPool2DLayer), ('hidden4', layers.DenseLayer), ('hidden5', layers.DenseLayer), ('output', layers.DenseLayer),], input_shape=(None, 1, 1, num_features), conv1_num_filters=32, conv1_filter_size=(1, 3), pool1_ds=(1, 2), conv2_num_filters=64, conv2_filter_size=(1, 2), pool2_ds=(1, 2), conv3_num_filters=128, conv3_filter_size=(1, 2), pool3_ds=(1, 2), hidden4_num_units=500, hidden5_num_units=500, output_num_units=num_classes, output_nonlinearity=softmax, update_learning_rate=0.01, update_momentum=0.9, max_epochs=100, verbose=1, ) ##X = X.reshape(-1, 1, 1, num_features) ##X_test = X_test.reshape(-1, 1, 1, num_features) net0.fit(X, y) make_submission(net0, X_test, ids, encoder)
lr = 1e-4 momentum = 0.9 net = NeuralNet( layers=[ ('i', layers.InputLayer), ('c1', Conv2DLayer), ('p1', MaxPool2DLayer), ('o', layers.DenseLayer), ], i_shape=(None, ) + ds.X.shape[1:], c1_num_filters=3, c1_filter_size=(3, 3), p1_pool_size=(2, 2), o_num_units=len(set(ds.y)), o_nonlinearity=nonlinearities.softmax, update_learning_rate=theano.shared(np.float32(lr)), update_momentum=theano.shared(np.float32(momentum)), regression=False, on_epoch_finished=[ AdjustVariable('update_learning_rate', start=lr, stop=0.0001), AdjustVariable('update_momentum', start=momentum, stop=0.999), EarlyStopping(patience=200), ], max_epochs=3000, verbose=1, ) net.fit(ds.X, ds.y) save_net(net)
regression=False, #loss=lasagne.objectives.multinomial_nll, use_label_encoder=True, batch_iterator_train=FlipBatchIterator(batch_size=256), on_epoch_finished=[ AdjustVariable('update_learning_rate', start=0.01, stop=0.0001), AdjustVariable('update_momentum', start=0.9, stop=0.999), EarlyStopping(patience=50), ], max_epochs=500, verbose=2, ) if TRAIN: net2.fit(X, y) with open(NETWORK, 'wb') as f: pickle.dump(net2, f, -1) def predict(): with open(NETWORK, 'rb') as f: model = pickle.load(f) testdata = load_test() predictions = np.zeros((testdata.shape[0], 121)) for i in range(0, testdata.shape[0], 4075): preds = model.predict_proba(testdata[i:i+4075,:].astype(np.float32).reshape(-1, 1, IMG_SIZE, IMG_SIZE)) predictions[i:i+4075,:] = preds print "Done: ", i+4075 return model, predictions
dense3_nonlinearity=nonlinearities.sigmoid, #dense4_nonlinearity = nonlinearities.sigmoid, output_nonlinearity=softmax, update=nesterov_momentum, update_learning_rate=0.03, eval_size=0.2, verbose=1, max_epochs=140, regression=False) clf = NeuralNet(**params) print "PARAMS : " print params clf.fit(X, Y) file_valid = pd.read_hdf("test.h5", "test") file_valid = np.array(file_valid) data = np.array([row[0:] for row in file_valid]) data = skpre.StandardScaler().fit_transform(data) number = range(45324, 53461) predict = clf.predict(data) with open('result.csv', 'wb') as csvfile: fieldnames = ['Id', 'y'] writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
# check if a trained model already exists cnn.load_params_from('./data_cache/combined_dataset/cnn_' + CNNCode + '-' + TrainCode + '.pkl') else: cnn_train = time.time() # training a new model for epoch in range(Epochs): # for every epoch for batch in patches_extract_all(Train): # for every batch inputs, targets = batch # data augmentation inputs, targets = data_aug(inputs, targets) # run cnn.fit for 1 iteration #cnn_fit = time.time(); cnn.fit(inputs, targets.reshape((-1, 1 * 32 * 32))) # print 'fitting cnn took: ', time.time()-cnn_fit, 'sec.'; # for every 10 epoch, print testing accuracy '''if epoch % 10 == 0: for batch in patches_extract_all(Test): inputs, targets = batch; predicts = cnn.predict(inputs); T = targets.reshape((-1, 1 * 32 * 32)).flatten().astype(np.int32); P = (predicts.flatten() > 0.5).astype(np.int32); print("======================= {:.4f} =======================".format(accuracy_score(T, P)));''' print('training cnn took: ', time.time() - cnn_train, 'sec.') # save the trained model cnn.save_params_to('./data_cache/combined_dataset/cnn_' + CNNCode + '-' + TrainCode + '.pkl')
cons = ({'type':'eq','fun':lambda w: 1-sum(w)}) bounds = [(0,1)]*6 weights = [] for train_index, test_index in re: aucList = [] predictions = [] gbm = xgb.train(params,xgb.DMatrix(result.iloc[train_index,:],y.iloc[train_index]),num_trees) rf.fit(X_train[train_index,:],y[train_index]) ada.fit(X_train[train_index,:],y[train_index]) svr.fit(X_train[train_index,:],y[train_index]) knn.fit(X_train[train_index,:],y[train_index]) y_train = encoder.fit_transform(y[train_index]).astype(np.int32) net0.fit(X_train[train_index,:],y_train) predictions.append(svr.predict_proba(X_train[test_index])[:,1]) predictions.append(gbm.predict(xgb.DMatrix(result.iloc[test_index]))) predictions.append(ada.predict_proba(X_train[test_index])[:,1]) predictions.append(rf.predict_proba(X_train[test_index])[:,1]) predictions.append(knn.predict_proba(X_train[test_index])[:,1]) predictions.append(net0.predict_proba(X_train[test_index])[:,1]) netAccuracy.append(roc_auc_score(y.iloc[test_index],net0.predict_proba(X_train[test_index])[:,1])) starting_values = [1./6]*len(predictions) # train_eval_probs = 0.30*svr.predict_proba(X_train[test_index])[:,1] + 0.15*gbm.predict(xgb.DMatrix(result.iloc[test_index])) \ # + 0.15*ada.predict_proba(X_train[test_index])[:,1] + 0.35*rf.predict_proba(X_train[test_index])[:,1] \ # + 0.05*knn.predict_proba(X_train[test_index])[:,1] res = minimize(log_loss_func, starting_values, method='SLSQP', bounds=bounds, constraints=cons) print('Ensamble Score: {best_score}'.format(best_score=res['fun'])) print('Best Weights: {weights}'.format(weights=res['x']))
def run_cross_validation(nfolds=10): unique_drivers = trainingLabels['subject'].unique() #driver_id = {} #for d_id in unique_drivers: #driver_id[d_id] = trainingLabels.loc[trainingLabels['subject'] == d_id]['img'] driver_id = trainingLabels['subject'] #print('Unique drivers ' + str(unique_drivers) + '\n' + str(driver_id)) kf = cross_validation.KFold(len(unique_drivers), n_folds=nfolds, shuffle=True) num_fold = 1 yfull_train = dict() yfull_test = [] for train_drivers, test_drivers in kf: unique_list_train = [unique_drivers[i] for i in train_drivers] #print('Unique drivers train ' + str(unique_list_train)) X_train, Y_train, train_index = copy_selected_drivers(xTrain, y, driver_id, unique_list_train) unique_list_valid = [unique_drivers[i] for i in test_drivers] #print('Unique drivers validation ' + str(unique_list_valid)) X_valid, Y_valid, test_index = copy_selected_drivers(xTrain, y, driver_id, unique_list_valid) outputLayer = None if source == '1': outputLayer = getNet1() #outputLayer = getNet2() #outputLayer = getNet8() numberEpochs = 30 elif source == '2': outputLayer = getNet3() numberEpochs = 30 elif source == '3': outputLayer = getNet4() #outputLayer = getNet5() #outputLayer = getNet6() #outputLayer = getNet7() numberEpochs = 10 elif source == '4': outputLayer = getNet9() #outputLayer = getNet10() numberEpochs = 30 net = NeuralNet( layers = outputLayer, update=updates.nesterov_momentum, #update=updates.adam, #update=updates.rmsprop, #update=updates.adadelta, update_learning_rate = 0.001, #update_beta1 = 0.9, #update_beta2 = 0.999, #update_epsilon = 1e-8, update_momentum = 0.9, #update_rho=0.95, #update_epsilon=1e-06, objective_loss_function = objectives.categorical_crossentropy, #objective=objectives.categorical_crossentropy, batch_iterator_train = BatchIterator(batch_size = batchSize), batch_iterator_test = BatchIterator(batch_size = batchSize), #custom_scores = [objectives.categorical_accuracy], use_label_encoder = True, #use_label_encoder = False, regression = False, max_epochs = numberEpochs, verbose = 1 ) net.fit(X_train, Y_train) predictValidY = net.predict_proba(X_valid) score = metrics.log_loss(Y_valid, predictValidY) print('Fold ' + str(num_fold) + ' score ' + str(score)) for i in range(len(test_index)): yfull_train[test_index[i]] = predictValidY[i] test_prediction = net.predict_proba(xTest) yfull_test.append(test_prediction) if gcEnabled: gc.collect() num_fold += 1 score = metrics.log_loss(y, dict_to_list(yfull_train)) print('Final log loss ' + str(score)) test_res = merge_several_folds_mean(yfull_test, nfolds) predictions = pd.DataFrame(test_res, index=files) predictions.index.name = 'img' predictions.columns = ['c0', 'c1', 'c2', 'c3', 'c4', 'c5', 'c6', 'c7', 'c8', 'c9'] predictions.to_csv(predictionsFile)
dropout2_p = 0.5, # output layer descriptors output_nonlinearity = lasagne.nonlinearities.softmax, output_num_units=10, #optimization parameters update=nesterov_momentum, update_learning_rate=0.01, max_epochs=max_epochs ) # In[10]: print('training') nn = net.fit(X_train, Y_train) print('training complete') # In[11]: # In[12]: X_test = np.load('./data/test_inputs.npy') # In[13]: X_test = X_test.reshape(-1, 1, PIXELS, PIXELS)
('dropout4', DropoutLayer), ('output', DenseLayer), ], input_shape=(None, 1, 28, 28), conv1_num_filters=32, conv1_filter_size=(3, 3), pool1_pool_size=(2, 2), dropout1_p=0.2, conv2_num_filters=64, conv2_filter_size=(3, 3), pool2_pool_size=(2, 2), dropout2_p=0.2, hidden3_num_units=1024, dropout3_p=0.5, hidden4_num_units=1024, dropout4_p=0.5, output_num_units=num_classes, output_nonlinearity=softmax, update=adagrad, update_learning_rate=theano.shared(float32(0.03)), eval_size=0.01, on_epoch_finished=[ AdjustVariable('update_learning_rate', start=0.01, stop=0.0001), ], max_epochs=30, verbose=1, ) net.fit(X, y) nombre = "LasagneCNN.csv" make_submission(net, X_test, encoder, nombre)
hidden_num_units=100, # number of units in hidden layer output_nonlinearity=None, # output layer uses identity function output_num_units=30, # 30 target values # optimization method: update=nesterov_momentum, update_learning_rate=0.01, update_momentum=0.9, regression=True, # flag to indicate we're dealing with regression problem max_epochs=400, # we want to train this many epochs verbose=1, ) X, y = load() net1.fit(X, y) def plot_sample(x, y, axis): # plotting the images with predicted keypoints img = x.reshape(96, 96) axis.imshow(img, cmap='gray') axis.scatter(y[0::2] * 48 + 48, y[1::2] * 48 + 48, marker='x', s=10) X, _ = load(test=True) y_pred = net1.predict(X) fig = pyplot.figure(figsize=(6, 6)) fig.subplots_adjust( left=0, right=1, bottom=0, top=1, hspace=0.05, wspace=0.05) for i in range(16): ax = fig.add_subplot(4, 4, i + 1, xticks=[], yticks=[])
def NNet(X,y,test): #rows = int(X.shape[0]) cols = int(X.shape[1]) net = NeuralNet( layers = [ ('input',layers.InputLayer), ('hidden1',layers.DenseLayer), ('dropout1',layers.DropoutLayer), ('hidden2',layers.DenseLayer), ('dropout2',layers.DropoutLayer), ('hidden3',layers.DenseLayer), ('dropout3',layers.DropoutLayer), ('hidden4',layers.DenseLayer), #('dropout4',layers.DropoutLayer), ('output',layers.DenseLayer), ], input_shape = (None,cols), hidden1_num_units = 800, dropout1_p = 0.4, hidden2_num_units = 500, dropout2_p = 0.3, hidden3_num_units = 300, dropout3_p = 0.3, hidden4_num_units = 200, #dropout4_p = 0.2, output_num_units = len(np.unique(y)), output_nonlinearity = lasagne.nonlinearities.softmax, update=nesterov_momentum, update_learning_rate=0.0001, update_momentum=0.9, max_epochs = 300, verbose = 1, ) # net.load_params_from('w3') # details = net.get_all_params() # oldw = net.get_all_params_values() ''' skf = cross_validation.StratifiedKFold(y,n_folds = 7) blend_train = np.zeros(X.shape[0]) prediction = [] blend_test_j = np.zeros((test.shape[0], len(skf))) for i,(train_index,cv_index) in enumerate(skf): print "Fold:",i X_train = X[train_index,] y_train = y[train_index] X_cv = X[cv_index,] #y_cv = y[cv_index] net.fit(X_train,y_train) blend_train[cv_index] = net.predict_proba(X_cv)[:,1] blend_test_j[:,i] = net.predict_proba(test)[:,1] prediction = blend_test_j.mean(1) ''' net.fit(X,y) prediction = net.predict_proba(test) return prediction
dense_nonlinearity=lasagne.nonlinearities.rectify, # dropout2 dropout2_p=0.5, # output output_nonlinearity=lasagne.nonlinearities.softmax, output_num_units=26, # optimization method params update=nesterov_momentum, update_learning_rate=0.02, update_momentum=0.9, max_epochs=50, verbose=1, ) # Train the network nn = net1.fit(X_train_s, y_train_s) #nn = net1.fit(X_train, y_train) # Test CNN preds = net1.predict(X_test) # Confusion matrix plot_confusion_matrix(y_test, preds) nn = net1.fit(X_train, y_train) # Test CNN preds = net1.predict(X_test) # Confusion matrix plot_confusion_matrix(y_test, preds)
conv3_num_filters=128, conv3_filter_size=(2, 2), pool3_pool_size=(2, 2), hidden4_num_units=500, hidden5_num_units=500, output_num_units=30, output_nonlinearity=None, update_learning_rate=theano.shared(float32(0.03)), update_momentum=theano.shared(float32(0.9)), regression=True, batch_iterator_train=FlipBatchIterator(batch_size=128), on_epoch_finished=[ AdjustVariable('update_learning_rate', start=0.03, stop=0.0001), AdjustVariable('update_momentum', start=0.9, stop=0.999), ], max_epochs=3000, verbose=1, ) ##################### ##################### ##################### X, y = load2d() # load 2-d data net5.fit(X, y) import cPickle as pickle with open('net5.pickle', 'wb') as f: pickle.dump(net5, f, -1)
def smart_find(X, y, X_valid, y_valid): loss = [] kf = KFold(n_splits=5, shuffle=True) conf_set = set() step = (64 + 10) / 4 max_neuron_units = step * 8 for i in range(1, max_neuron_units, step): for j in range(0, max_neuron_units, step): for k in range(0, max_neuron_units, step): struct_net = (i) l = InputLayer(shape=(None, X.shape[1])) # ------- HIDDEN ----------- l = DenseLayer(l, num_units=i, nonlinearity=softmax) if j > 0: if i + step < j: continue l = DenseLayer(l, num_units=j, nonlinearity=softmax) struct_net = (i, j) if k > 0: if i + step < k or j + step < k: continue struct_net = (i, j, k) l = DenseLayer(l, num_units=k, nonlinearity=softmax) # ------- HIDDEN ----------- l = DenseLayer(l, num_units=len(np.unique(y)), nonlinearity=softmax) net = NeuralNet(l, update=adam, update_learning_rate=0.01, max_epochs=250) if struct_net in conf_set: continue print('=' * 40) print(struct_net) print('=' * 40) conf_set.add(struct_net) k_loss = [] y_data = np.array([y]).transpose() data = np.concatenate((X, y_data), axis=1) for train_index, test_index in kf.split(data): X_train, X_test = X[train_index], X[test_index] y_train, y_test = y[train_index], y[test_index] net.fit(X_train, y_train) y_pred = net.predict(X_test) loss_error = net.score(X_test, y_test) # loss_error = mean_squared_error(y_test, y_pred) k_loss.append(loss_error) print(loss_error) loss_net = (i, j, k, np.array(k_loss).mean()) print(loss_net) loss.append(loss_net) print('=' * 40) # for i in range(1, max_hidden_layers): # for j in range((64 + 10) // 2 // i, max_neuron_units // i, 10 // i): # print('=' * 40) # print('%s hidden layers' % i) # print('%s neurons' % j) # print('=' * 40) # l = InputLayer(shape=(None, X.shape[1])) # for k in range(i): # l = DenseLayer(l, num_units=j, nonlinearity=softmax) # l = DenseLayer(l, num_units=len(np.unique(y)), nonlinearity=softmax) # net = NeuralNet(l, update=adam, update_learning_rate=0.01, max_epochs=500) # # k_loss = [] # y_data = np.array([y]).transpose() # data = np.concatenate((X, y_data), axis=1) # for train_index, test_index in kf.split(data): # X_train, X_test = X[train_index], X[test_index] # y_train, y_test = y[train_index], y[test_index] # # net.fit(X_train, y_train) # y_pred = net.predict(X_test) # loss_error = mean_squared_error(y_test, y_pred) # k_loss.append(loss_error) # print(loss_error) # # loss_net = (i, j, np.array(k_loss).mean()) # print(loss_net) # loss.append(loss_net) # print('=' * 40) print(min(loss, key=lambda x: x[3])) print(max(loss, key=lambda x: x[3])) print(loss)
class ClassificationLinearBlending(ClassficationBase.ClassificationBase): def __init__(self, isTrain, isOutlierRemoval=0): super(ClassificationLinearBlending, self).__init__(isTrain, isOutlierRemoval) # data preprocessing self.dataPreprocessing() # create logistic regression object self.logreg = linear_model.LogisticRegression(tol=1e-6, penalty='l1', C=0.0010985411419875584) # create adaboost object self.dt_stump = DecisionTreeClassifier(max_depth=10) self.ada = AdaBoostClassifier( base_estimator=self.dt_stump, learning_rate=1, n_estimators=5, algorithm="SAMME.R") # create knn object self.knn = neighbors.KNeighborsClassifier(6, weights='uniform') # create decision tree object self.decisiontree = DecisionTreeClassifier(max_depth=50) # create neural network object self.net1 = NeuralNet( layers=[ # three layers: one hidden layer ('input', layers.InputLayer), ('hidden', layers.DenseLayer), #('hidden2', layers.DenseLayer), ('output', layers.DenseLayer), ], # layer parameters: input_shape=(None, 12), # inut dimension is 12 hidden_num_units=6, # number of units in hidden layer #hidden2_num_units=3, # number of units in hidden layer output_nonlinearity=lasagne.nonlinearities.sigmoid, # output layer uses sigmoid function output_num_units=1, # output dimension is 1 # optimization method: update=nesterov_momentum, update_learning_rate=0.002, update_momentum=0.9, regression=True, # flag to indicate we're dealing with regression problem max_epochs=25, # we want to train this many epochs verbose=0, ) def dataPreprocessing(self): # deal with unbalanced data self.dealingUnbalancedData() # Standardization #self.Standardization() def training(self): # train the models self.logreg.fit(self.X_train, self.y_train.ravel()) self.ada.fit(self.X_train, self.y_train.reshape((self.y_train.shape[0], ))) self.knn.fit(self.X_train, self.y_train.ravel()) self.decisiontree.fit(self.X_train, self.y_train) self.net1.fit(self.X_train, self.y_train) def predict(self): # predict the test data y_pred1 = self.logreg.predict(self.X_test) y_pred1 = y_pred1.reshape((y_pred1.shape[0], 1)) y_pred2 = self.ada.predict(self.X_test) y_pred2 = y_pred2.reshape((y_pred2.shape[0], 1)) y_pred3 = self.knn.predict(self.X_test) y_pred3 = y_pred3.reshape((y_pred3.shape[0], 1)) y_pred4 = self.decisiontree.predict(self.X_test) y_pred4 = y_pred4.reshape((y_pred4.shape[0], 1)) # predict neural network # predict the test data y_pred_train = self.net1.predict(self.X_train) y_pred5 = self.net1.predict(self.X_test) # keep all the predictions y_preds = [] y_preds.append(y_pred1) y_preds.append(y_pred2) y_preds.append(y_pred3) y_preds.append(y_pred4) y_preds.append(y_pred5) # 1 for buy, 0 for wait median = np.median(y_pred_train) mean = np.mean(y_pred_train) y_pred5[y_pred5>=median] = 1 # change this threshold y_pred5[y_pred5<median] = 0 y_pred5 = y_pred5.reshape((y_pred5.shape[0], 1)) # get the error rate self.y_pred = (y_pred2+y_pred3+y_pred4)/3 self.y_pred[self.y_pred >= 0.5] = 1 self.y_pred[self.y_pred < 0.5] = 0 e1 = 1 - np.sum(self.y_test == y_pred1) * 1.0 / y_pred1.shape[0] e2 = 1 - np.sum(self.y_test == y_pred2) * 1.0 / y_pred2.shape[0] e3 = 1 - np.sum(self.y_test == y_pred3) * 1.0 / y_pred3.shape[0] e4 = 1 - np.sum(self.y_test == y_pred4) * 1.0 / y_pred4.shape[0] e5 = 1 - np.sum(self.y_test == y_pred5) * 1.0 / y_pred5.shape[0] e = 1 - np.sum(self.y_test == self.y_pred) * 1.0 / self.y_pred.shape[0] print "e1 = {}".format(e1) print "e2 = {}".format(e2) print "e3 = {}".format(e3) print "e4 = {}".format(e4) print "e5 = {}".format(e5) print "Uniform error = {}".format(e) # keep all the error rates errors = [] errors.append(e1) errors.append(e2) errors.append(e3) errors.append(e4) errors.append(e5) # totalAlpha = 0 # self.y_pred = np.zeros(shape=self.y_pred.shape) # for i in range(5): # if errors[i] <= 0.5: # alpha = math.log(math.sqrt((1-errors[i])/errors[i])) # totalAlpha += alpha # self.y_pred = self.y_pred + alpha*y_preds[i] # # # predict the blending output # self.y_pred = self.y_pred / totalAlpha # self.y_pred[self.y_pred >= 0.5] = 1 # self.y_pred[self.y_pred < 0.5] = 0 alpha2 = math.log(math.sqrt((1-e2)/e2)) alpha3 = math.log(math.sqrt((1-e3)/e3)) alpha4 = math.log(math.sqrt((1-e4)/e4)) self.y_pred = (alpha2*y_pred2 + alpha3*y_pred3 + alpha4*y_pred4) * 1.0 /(alpha2+alpha3+alpha4)
def train(): weather = load_weather() training = load_training() X = assemble_X(training, weather) mean, std = normalize(X) y = assemble_y(training) col_labels = [ "date.year", "date.month", "date.day", "block", "lat", "long", ] for obs in [ "Tmax", "Tmin", "Tavg", "DewPoint", "WetBulb", "PrecipTotal", "Depart", "Sunrise", "Sunset", "Heat", "Cool", "ResultSpeed", "ResultDir" ]: for day in ["1", "2", "3", "5", "8", "12"]: col_labels.append(obs + "_" + day) for i_spec in range(6): col_labels.append("species_" + str(i_spec)) X_file = csv.writer(open("X.csv", "w")) X_file.writerow(col_labels) for row in X: X_file.writerow(row) y_file = csv.writer(open("y.csv", "w")) y_file.writerow(["y"]) for row in y: y_file.writerow(row) input_size = len(X[0]) learning_rate = theano.shared(np.float32(0.1)) net = NeuralNet( layers=[ ('input', InputLayer), ('hidden1', DenseLayer), ('dropout1', DropoutLayer), ('hidden2', DenseLayer), ('dropout2', DropoutLayer), ('output', DenseLayer), ], # layer parameters: input_shape=(None, input_size), hidden1_num_units=600, dropout1_p=0.5, hidden2_num_units=400, dropout2_p=0.5, output_nonlinearity=sigmoid, output_num_units=1, # optimization method: update=nesterov_momentum, update_learning_rate=learning_rate, update_momentum=0.9, # Decay the learning rate on_epoch_finished=[ AdjustVariable(learning_rate, target=0, half_life=20), ], # This is silly, but we don't want a stratified K-Fold here # To compensate we need to pass in the y_tensor_type and the loss. regression=True, y_tensor_type=T.imatrix, objective_loss_function=binary_crossentropy, max_epochs=500, eval_size=0.1, verbose=1, ) X, y = shuffle(X, y, random_state=123) net.fit(X, y) _, X_valid, _, y_valid = net.train_test_split(X, y, net.eval_size) probas = net.predict_proba(X_valid)[:, 0] print("ROC score", metrics.roc_auc_score(y_valid, probas)) return net, mean, std
update_momentum=theano.shared(np.float32(0.9)), # Decay the learning rate on_epoch_finished=[ AdjustVariable('update_learning_rate', start=0.001, stop=0.0001), AdjustVariable('update_momentum', start=0.9, stop=0.99), ], regression=True, y_tensor_type=T.imatrix, objective_loss_function=binary_crossentropy, #batch_iterator_train = BatchIterator(batch_size = 256), max_epochs=30, eval_size=0.0, verbose=2, ) seednumber = 1235 np.random.seed(seednumber) net.fit(train, labels) preds = net.predict_proba(test)[:, 0] submission = pd.read_csv('../input/sample_submission.csv') submission["TARGET"] = preds submission.to_csv('Lasagne_bench.csv', index=False)