def NNet(X, y, test): #rows = int(X.shape[0]) cols = int(X.shape[1]) net = NeuralNet( layers=[ ('input', layers.InputLayer), ('hidden1', layers.DenseLayer), ('dropout1', layers.DropoutLayer), ('hidden2', layers.DenseLayer), ('dropout2', layers.DropoutLayer), ('hidden3', layers.DenseLayer), ('dropout3', layers.DropoutLayer), ('hidden4', layers.DenseLayer), #('dropout4',layers.DropoutLayer), ('output', layers.DenseLayer), ], input_shape=(None, cols), hidden1_num_units=70, dropout1_p=0.4, hidden2_num_units=50, dropout2_p=0.3, hidden3_num_units=30, dropout3_p=0.3, hidden4_num_units=15, #dropout4_p = 0.2, output_num_units=len(np.unique(y)), output_nonlinearity=lasagne.nonlinearities.softmax, update=nesterov_momentum, update_learning_rate=0.01, update_momentum=0.9, max_epochs=100, verbose=1, ) # net.load_params_from('w3') # details = net.get_all_params() # oldw = net.get_all_params_values() skf = cross_validation.StratifiedKFold(y, n_folds=7) blend_train = np.zeros(X.shape[0]) prediction = [] blend_test_j = np.zeros((test.shape[0], len(skf))) for i, (train_index, cv_index) in enumerate(skf): print "Fold:", i X_train = X[train_index, ] y_train = y[train_index] X_cv = X[cv_index, ] #y_cv = y[cv_index] net.fit(X_train, y_train) blend_train[cv_index] = net.predict_proba(X_cv)[:, 1] blend_test_j[:, i] = net.predict_proba(test)[:, 1] prediction = blend_test_j.mean(1) return prediction
def fit(xTrain, yTrain, dense0_num=800, dropout_p=0.5, dense1_num=500, update_learning_rate=0.01, update_momentum=0.9, test_ratio=0.2, max_epochs=20): #update_momentum=0.9, test_ratio=0.2, max_epochs=20, train_fname='train.csv'): #xTrain, yTrain, encoder, scaler = load_train_data(train_fname) #xTest, ids = load_test_data('test.csv', scaler) num_features = len(xTrain[0,:]) num_classes = 9 print num_features layers0 = [('input', InputLayer), ('dense0', DenseLayer), ('dropout', DropoutLayer), ('dense1', DenseLayer), ('output', DenseLayer)] clf = NeuralNet(layers=layers0, input_shape=(None, num_features), dense0_num_units=dense0_num, dropout_p=dropout_p, dense1_num_units=dense1_num, output_num_units=num_classes, output_nonlinearity=softmax, update=nesterov_momentum, update_learning_rate=update_learning_rate, update_momentum=update_momentum, eval_size=test_ratio, verbose=1, max_epochs=max_epochs) clf.fit(xTrain, yTrain) ll_train = metrics.log_loss(yTrain, clf.predict_proba(xTrain)) print ll_train return clf
def nn_level2(train_x, train_y, test_x): ''' neural net proba predict for level 2 ''' num_classes = len(np.unique(train_y)) num_features = train_x.shape[1] layers0 = [('input', InputLayer), ('dropoutf', DropoutLayer), ('dense0', DenseLayer), ('dropout', DropoutLayer), ('dense1', DenseLayer), ('dropout2', DropoutLayer), ('output', DenseLayer)] net0 = NeuralNet( layers=layers0, input_shape=(None, num_features), dropoutf_p=0.15, dense0_num_units=1000, dropout_p=0.25, dense1_num_units=500, dropout2_p=0.25, output_num_units=num_classes, output_nonlinearity=softmax, update=adagrad, update_learning_rate=theano.shared(np.float32(0.01)), # on_epoch_finished=[AdjustVariable('update_learning_rate', start=0.02, stop=0.016)], max_epochs=18, eval_size=0.2, verbose=1, ) net0.fit(train_x, train_y) pred = net0.predict_proba(test_x).astype(np.float32) return pred
class DeepCls(object): def __init__(self,n_cats=2,input_size=1040,k=1): self.net1 = NeuralNet( layers=[ ('input', layers.InputLayer), ('hidden', layers.DenseLayer), ('output', layers.DenseLayer), ], input_shape=(None, input_size), hidden_num_units=300, output_nonlinearity=softmax, output_num_units=n_cats, update=nesterov_momentum, update_learning_rate=0.01, update_momentum=0.9, regression=False, max_epochs=2500, verbose=1, ) self.k=k def __call__(self,hog_desc): hog_desc=np.expand_dims(hog_desc,0) prob=self.net1.predict_proba(hog_desc) prob=prob.flatten() return prob[self.k]
def OptNN(d1, h1, d2, h2, d3, start, stop, max_epochs): params2 = params.copy() on_epoch = [AdjustVariable('update_learning_rate', start = start, stop = stop), AdjustVariable('update_momentum', start = .9, stop = .999)] params2['dropout1_p'] = d1 params2['dropout2_p'] = d2 params2['dropout3_p'] = d3 params2['dropout4_p'] = d4 params2['hidden1_num_units'] = h1 params2['hidden2_num_units'] = h2 params2['hidden3_num_units'] = h3 params2['max_epochs'] = max_epochs params2['on_epoch_finished'] = on_epoch kcv = StratifiedKFold(Y, 5, shuffle = True) res = np.empty((len(Y), len(np.unique(Y)))); i = 1 CVScores = [] for train_idx, valid_idx in kcv: logger.info("Running fold %d...", i); i += 1 net = NeuralNet(**params2) net.set_params(eval_size = None) net.fit(X[train_idx], Y[train_idx]) res[valid_idx, :] = net.predict_proba(X[valid_idx]) CVScores.append(log_loss(Y[valid_idx], res[valid_idx])) return -np.mean(CVScores)
def train(trainfile, weatherfile): weather = load_weather(weatherfile) training = load_training(trainfile) np.random.seed(42) X = assemble_X(training, weather) mean, std = normalize(X) y = assemble_y(training) input_size = len(X[0]) learning_rate = theano.shared(np.float32(0.1)) net = NeuralNet( layers=[ ('input', InputLayer), ('hidden1', DenseLayer), ('dropout1', DropoutLayer), ('hidden2', DenseLayer), ('dropout2', DropoutLayer), ('output', DenseLayer), ], # layer parameters: input_shape=(None, input_size), hidden1_num_units=400, dropout1_p=0.4, hidden2_num_units=200, dropout2_p=0.4, output_nonlinearity=sigmoid, output_num_units=1, # optimization method: update=nesterov_momentum, update_learning_rate=learning_rate, update_momentum=0.9, # Decay the learning rate on_epoch_finished=[ AdjustVariable(learning_rate, target=0, half_life=4), ], # This is silly, but we don't want a stratified K-Fold here # To compensate we need to pass in the y_tensor_type and the loss. regression=True, y_tensor_type=T.imatrix, objective_loss_function=binary_crossentropy, max_epochs=60, eval_size=0.1, verbose=1, ) X, y = shuffle(X, y, random_state=123) net.fit(X, y) _, X_valid, _, y_valid = net.train_test_split(X, y, net.eval_size) probas = net.predict_proba(X_valid)[:, 0] print("ROC score", metrics.roc_auc_score(y_valid, probas)) return net, mean, std
def train(): weather = load_weather() training = load_training() X = assemble_X(training, weather) print len(X[0]) mean, std = normalize(X) y = assemble_y(training) input_size = len(X[0]) learning_rate = theano.shared(np.float32(0.1)) net = NeuralNet( layers=[ ('input', InputLayer), ('hidden1', DenseLayer), ('dropout1', DropoutLayer), ('hidden2', DenseLayer), ('dropout2', DropoutLayer), ('output', DenseLayer), ], # layer parameters: input_shape=(None, input_size), hidden1_num_units=325, dropout1_p=0.4, hidden2_num_units=325, dropout2_p=0.4, output_nonlinearity=sigmoid, output_num_units=1, # optimization method: update=nesterov_momentum, update_learning_rate=learning_rate, update_momentum=0.9, # Decay the learning rate on_epoch_finished=[ AdjustVariable(learning_rate, target=0, half_life=1), ], # This is silly, but we don't want a stratified K-Fold here # To compensate we need to pass in the y_tensor_type and the loss. regression=True, y_tensor_type = T.imatrix, objective_loss_function = binary_crossentropy, max_epochs=85, eval_size=0.1, verbose=1, ) X, y = shuffle(X, y, random_state=123) net.fit(X, y) _, X_valid, _, y_valid = net.train_test_split(X, y, net.eval_size) probas = net.predict_proba(X_valid)[:,0] print("ROC score", metrics.roc_auc_score(y_valid, probas)) return net, mean, std
class Classifier(BaseEstimator): def __init__(self): self.net = None self.label_encoder = None def fit(self, X, y): layers0 = [('input', InputLayer), ('dropoutf', DropoutLayer), ('dense0', DenseLayer), ('dropout0', DropoutLayer), ('dense1', DenseLayer), ('dropout1', DropoutLayer), ('dense2', DenseLayer), ('dropout2', DropoutLayer), ('dense3', DenseLayer), ('dropout3', DropoutLayer), ('output', DenseLayer)] X = X.astype(theano.config.floatX) self.label_encoder = LabelEncoder() y = self.label_encoder.fit_transform(y).astype(np.int32) self.scaler = StandardScaler() X = self.scaler.fit_transform(X) num_classes = len(self.label_encoder.classes_) num_features = X.shape[1] self.net = NeuralNet(layers=layers0, input_shape=(None, num_features), dropoutf_p=0.15, dense0_num_units=1024, dropout0_p=0.5, dense0_nonlinearity=rectify, dense1_num_units=1024, dropout1_p=0.15, dense1_nonlinearity=rectify, dense2_num_units=1024, dropout2_p=0.15, dense2_nonlinearity=rectify, dense3_num_units=1024, dropout3_p=0.15, dense3_nonlinearity=rectify, output_num_units=num_classes, update=adagrad, update_learning_rate=0.01, eval_size=0.2, verbose=1, max_epochs=150) self.net.fit(X, y) return self def predict(self, X): X = X.astype(theano.config.floatX) X = self.scaler.fit_transform(X) return self.label_encoder.inverse_transform(self.net.predict(X)) def predict_proba(self, X): X = X.astype(theano.config.floatX) X = self.scaler.fit_transform(X) return self.net.predict_proba(X)
def OptNN2(d0, d1,d2, d3, h1, h2, h3, me, ls, le): h1, h2, h3 = int(h1), int(h2), int(h3); me = int(me) params = dict( layers = [ ('input', layers.InputLayer), ('dropout1', layers.DropoutLayer), ('hidden1', layers.DenseLayer), ('dropout2', layers.DropoutLayer), ('hidden2', layers.DenseLayer), ('dropout3', layers.DropoutLayer), ('hidden3', layers.DenseLayer), ('dropout4', layers.DropoutLayer), ('output', layers.DenseLayer), ], input_shape = (None, 93), dropout1_p = d0, hidden1_num_units = h1, dropout2_p = d1, hidden2_num_units = h2, dropout3_p = d2, hidden3_num_units = h3, dropout4_p = d3, output_nonlinearity = softmax, output_num_units = 9, update = nesterov_momentum, update_learning_rate = theano.shared(float32(l_start)), update_momentum = theano.shared(float32(m_start)), regression = False, on_epoch_finished = [ AdjustVariable('update_learning_rate', start = ls, stop = le, is_log = True), AdjustVariable('update_momentum', start = m_start, stop = m_stop, is_log = False), ], max_epochs = me, verbose = 1, ) CVScores = [] res = np.empty((len(Y), len(np.unique(Y)))) kcv = StratifiedKFold(Y, 5, shuffle = True); i = 1 for train_idx, valid_idx in kcv: logger.info("Running fold %d...", i); i += 1 net = NeuralNet(**params) net.set_params(eval_size = None) net.fit(X[train_idx], Y[train_idx]) res[valid_idx, :] = net.predict_proba(X[valid_idx]) CVScores.append(log_loss(Y[valid_idx], res[valid_idx])) return -np.mean(CVScores)
def train_test(train, labels, test, weight_decay): net = NeuralNet( layers=[ ('input', InputLayer), ('dropout0', DropoutLayer), ('dense1', DenseLayer), ('dropout1', DropoutLayer), ('dense2', DenseLayer), ('dropout2', DropoutLayer), ('dense3', DenseLayer), ('dropout3', DropoutLayer), ('output', DenseLayer), ], update=nesterov_momentum, loss=None, objective=partial(WeightDecayObjective, weight_decay=weight_decay), regression=False, max_epochs=600, eval_size=0.1, #on_epoch_finished = None, #on_training_finished = None, verbose=bool(VERBOSITY), input_shape=(None, train.shape[1]), output_num_units=NCLASSES, dense1_num_units=700, dense2_num_units=1000, dense3_num_units=700, dense1_nonlinearity=LeakyRectify(leakiness=0.1), dense2_nonlinearity=LeakyRectify(leakiness=0.1), dense3_nonlinearity=LeakyRectify(leakiness=0.1), output_nonlinearity=softmax, dense1_W=HeUniform(), dense2_W=HeUniform(), dense3_W=HeUniform(), dense1_b=Constant(0.), dense2_b=Constant(0.), dense3_b=Constant(0.), output_b=Constant(0.), dropout0_p=0.1, dropout1_p=0.6, dropout2_p=0.6, dropout3_p=0.6, update_learning_rate=shared(float32(0.02)), # update_momentum=shared(float32(0.9)), # batch_iterator_train=BatchIterator(batch_size=128), batch_iterator_test=BatchIterator(batch_size=128), ) net.fit(train, labels) return net.predict_proba(test)
def train(hidden_units, subimission=False): X_train, X_test, y_train, y_test = cross_validation.train_test_split( X, y, test_size=0.2, random_state=561672) if subimission: X_train = X y_train = y layers0 = [('input', InputLayer), ('dense0', DenseLayer), ('gausian0', GaussianNoiseLayer), ('dense1', DenseLayer), ('dropout0', DropoutLayer), ('dense2', DenseLayer), ('output', DenseLayer)] net0 = NeuralNet(layers=layers0, input_shape=(None, num_features), dense0_num_units=hidden_units[0], dense1_num_units=hidden_units[1], dense2_num_units=hidden_units[2], dropout0_p=0.5, output_num_units=num_classes, output_nonlinearity=softmax, update=nesterov_momentum, update_learning_rate=theano.shared(float32(0.03)), update_momentum=theano.shared(float32(0.9)), on_epoch_finished=[ AdjustVariable('update_learning_rate', start=0.03, stop=0.0001), AdjustVariable('update_momentum', start=0.9, stop=0.999), ], eval_size=0.2, verbose=1, max_epochs=15) net0.fit(X_train, y_train) if subimission: make_submission(net0, X_test_submission, ids, encoder, name="nn_%f_%f.csv" % (hidden_units[0], hidden_units[1])) else: y_prob = net0.predict_proba(X_test) print(" -- Finished training.") score = logloss_mc(y_test, y_prob) print("Score %f" % score) return score, net0
def fit_two_dense(xTrain, yTrain, dropout_in=0.2, dense0_num=800, dropout_p=0.5, dense1_num=500, update_learning_rate=0.008, update_momentum=0.9, test_ratio=0.2, max_epochs=40): #xTrain, yTrain, encoder, scaler = load_train_data(train_fname) #xTest, ids = load_test_data('test.csv', scaler) #xTrain, yTrain, encoder, scaler = load_train_data(train_fname) #xTest, ids = load_test_data('test.csv', scaler) num_features = len(xTrain[0, :]) num_classes = 9 print num_features layers0 = [('input', InputLayer), ('dropoutin', DropoutLayer), ('dense0', DenseLayer), ('dropout', DropoutLayer), ('dense1', DenseLayer), ('output', DenseLayer)] clf = NeuralNet(layers=layers0, input_shape=(None, num_features), dropoutin_p=dropout_in, dense0_num_units=dense0_num, dropout_p=dropout_p, dense1_num_units=dense1_num, output_num_units=num_classes, output_nonlinearity=softmax, update=nesterov_momentum, update_learning_rate=update_learning_rate, update_momentum=update_momentum, eval_size=test_ratio, verbose=1, max_epochs=max_epochs) clf.fit(xTrain, yTrain) ll_train = metrics.log_loss(yTrain, clf.predict_proba(xTrain)) print ll_train return clf
def calc_prob_bag(i, best_max_epochs, NNargs, X_all, y_all, X_test): np.random.seed(111*(i+1)) # diffs random seed to diffs bagging print('\n - Bag: %i ' % (i+1)) if best_max_epochs == 0: print(' First fit to get optimal num of epochs...') NNargs["max_epochs"] = 1000 NNargs["eval_size"] = 0.05 # just a small test set to derive optimal numepochs NNargs["on_epoch_finished"][-1] = EarlyStopping(patience=25) # more patience clf_bag = NeuralNet(**NNargs) clf_bag.fit(X_all, y_all) global GLOBrealnumepochs best_max_epochs = GLOBrealnumepochs print(' we will refit now with max epochs = %i' % best_max_epochs) NNargs["max_epochs"] = best_max_epochs NNargs["eval_size"] = 0.0001 NNargs["on_epoch_finished"][-1] = EarlyStopping(patience=1000) # kind of a infinite patience to let max epochs rule clf_bag = NeuralNet(**NNargs) clf_bag.fit(X_all, y_all) probs_bags = clf_bag.predict_proba(X_test) return probs_bags
def main(deepL, alignments): S, X = make_xy(alignments, deepL['features']) h = DictVectorizer(sparse=False) X = h.fit_transform(X) X = X.astype(floatX) # normalize feature matrix min_max_scaler = preprocessing.MinMaxScaler() X = min_max_scaler.fit_transform(X) clf = NeuralNet( layers=model(deepL['input_nodes'], deepL['output_nodes']), update=nesterov_momentum, update_learning_rate=0.01, update_momentum=0.9, regression=False, max_epochs=100, verbose=2, ) clf.load_params_from(opt.path + "/model/model.pkl") proba = clf.predict_proba(X) preds = [] probs = [] deepL['Y_rev'].update({10000: "unclassified"}) for ipx, p in enumerate(proba): px = np.argmax(p) if p[px] > min_prob[deepL['Y_rev'][px]]: preds.append(px) probs.append(p[px]) else: preds.append(10000) probs.append(0) return [[S[ix], deepL['Y_rev'][i], probs[ix]] for ix, i in enumerate(preds)]
def train_dnn(train, train_y, test): num_features = train.shape[1] num_classes = len(list(set(train_y))) layers0 = [('input', InputLayer), ('dropout0', DropoutLayer), ('dense0', DenseLayer), ('dropout1', DropoutLayer), ('dense1', DenseLayer), ('dropout2', DropoutLayer), ('output', DenseLayer)] net0 = NeuralNet(layers=layers0, input_shape=(None, num_features), dropout0_p = 0.1, #theano.shared(float32(0.1)), dense0_num_units= 5000, dropout1_p= 0.3, #theano.shared(float32(0.5)), dense1_num_units = 10000, dropout2_p = 0.5, output_num_units=num_classes, output_nonlinearity=softmax, update=nesterov_momentum, #update_learning_rate=0.003, #update_momentum=0.9, update_learning_rate = theano.shared(float32(0.001)), update_momentum=theano.shared(float32(0.9)), objective_loss_function = categorical_crossentropy, train_split = TrainSplit(0.2), verbose=1, max_epochs=150, on_epoch_finished=[ EarlyStopping(patience = 20), AdjustVariable('update_learning_rate', start=0.001, stop=0.0001), AdjustVariable('update_momentum', start=0.9, stop=0.999), ] ) net0.fit(train, train_y) print('Prediction Complete') pred1 = net0.predict_proba(test) return pred1
def train(features, features_sub,label): layers0 = [('input', InputLayer), ('dropout0', DropoutLayer), ('dense0', DenseLayer), ('dropout1', DropoutLayer), ('dense1', DenseLayer), ('dropout2', DropoutLayer), ('output', DenseLayer)] net0 = NeuralNet(layers=layers0, input_shape=(None, features.shape[1]), dropout0_p = 0.18, #theano.shared(float32(0.1)), dense0_num_units= 2000, dropout1_p= 0.6, #theano.shared(float32(0.4)), dense1_num_units= 4000, dropout2_p = 0.9, #theano.shared(float32(0.7)), output_num_units= len(set(label)), output_nonlinearity=softmax, update=nesterov_momentum, update_momentum = 0.95, #theano.shared(float32(0.9)), update_learning_rate = 0.002, #theano.shared(float32(0.01)), #update_momentum=theano.shared(float32(0.9)), train_split = TrainSplit(0.2), verbose=1, max_epochs = 40 # on_epoch_finished=[ # AdjustVariable('update_learning_rate', start=0.01, stop=0.005), # #AdjustVariable('update_momentum', start=0.9, stop=0.999), # #AdjustVariable('dropout0_p', start = 0.1, stop = 0.2), # # AdjustVariable('dropout1_p', start = 0.45, stop = 0.6), # # AdjustVariable('dropout2_p', start = 0.8, stop = 0.9) # ] ) features = np.array(features.values, dtype = np.float32) net0.fit(features, np.array(label, dtype = np.int32) ) print('Prediction Complete') results = net0.predict_proba(np.array(features_sub.values, dtype = np.float32)) return
def calc_prob_bag(i, best_max_epochs, NNargs, X_all, y_all, X_test): np.random.seed(111 * (i + 1)) # diffs random seed to diffs bagging print('\n - Bag: %i ' % (i + 1)) if best_max_epochs == 0: print(' First fit to get optimal num of epochs...') NNargs["max_epochs"] = 1000 NNargs[ "eval_size"] = 0.05 # just a small test set to derive optimal numepochs NNargs["on_epoch_finished"][-1] = EarlyStopping( patience=25) # more patience clf_bag = NeuralNet(**NNargs) clf_bag.fit(X_all, y_all) global GLOBrealnumepochs best_max_epochs = GLOBrealnumepochs print(' we will refit now with max epochs = %i' % best_max_epochs) NNargs["max_epochs"] = best_max_epochs NNargs["eval_size"] = 0.0001 NNargs["on_epoch_finished"][-1] = EarlyStopping( patience=1000) # kind of a infinite patience to let max epochs rule clf_bag = NeuralNet(**NNargs) clf_bag.fit(X_all, y_all) probs_bags = clf_bag.predict_proba(X_test) return probs_bags
def classify(train_file, test_file, output_file): test = pd.read_csv(test_file) train = pd.read_csv(train_file) test['age'] = test['age'] / train['age'].max() train['age'] = train['age'] / train['age'].max() train.replace(to_replace={'Transfer': 0, 'Return_to_owner': 1, 'Euthanasia': 2, 'Adoption': 3, 'Died': 4}, inplace=True) test.replace(to_replace={'Transfer': 0, 'Return_to_owner': 1, 'Euthanasia': 2, 'Adoption': 3, 'Died': 4}, inplace=True) train_X = train.ix[:, features].as_matrix().astype('float32') train_Y = train.ix[:, 'outcome'].as_matrix().astype('int32') test_X = test.ix[:, features].as_matrix().astype('float32') model = NeuralNet( layers=[ ('input', layers.InputLayer), ('hidden1', layers.DenseLayer), ('hidden2', layers.DenseLayer), ('output', layers.DenseLayer), ], input_shape=(None, len(features)), hidden1_num_units=100, hidden1_nonlinearity=sigmoid, hidden2_num_units=100, hidden2_nonlinearity=rectify, max_epochs=100, output_nonlinearity=softmax, output_num_units=5, update_learning_rate=0.01, ).fit(train_X, train_Y) all_proba = model.predict_proba(test_X).reshape(test_X.shape[0], 5) result = pd.DataFrame( data=all_proba, columns=['Transfer', 'Return_to_owner', 'Euthanasia', 'Adoption', 'Died'], index=test['id'] ) result.to_csv(output_file, index_label="ID", float_format='%.5f')
def train_autoencoder(train, train_y, test): num_features = train.shape[1] net = NeuralNet( layers=[ ('input', InputLayer), ('auto', AutoEncoder), ('output', DenseLayer), ], input_shape=(None, 1, num_features), auto_num_units = 1000, auto_n_hidden = 10, output_num_units=1000, update_learning_rate=theano.shared(float32(0.03)), update_momentum=theano.shared(float32(0.9)), output_nonlinearity=nonlinearities.softmax, regression=True, max_epochs=3, verbose=1, ) net.fit(train, train_y) with open('net.pickle', 'wb') as f: pickle.dump(net, f, -1) pred_auto = net.predict_proba(test) return pred_auto
def nn_level2(train_x, train_y, test_x): ''' neural net proba predict for level 2 ''' num_classes = len(np.unique(train_y)) num_features = train_x.shape[1] layers0 = [('input', InputLayer), ('dropoutf', DropoutLayer), ('dense0', DenseLayer), ('dropout', DropoutLayer), ('dense1', DenseLayer), ('dropout2', DropoutLayer), ('output', DenseLayer)] net0 = NeuralNet(layers=layers0, input_shape=(None, num_features), dropoutf_p=0.15, dense0_num_units=1000, dropout_p=0.25, dense1_num_units=500, dropout2_p=0.25, output_num_units=num_classes, output_nonlinearity=softmax, update=adagrad, update_learning_rate=theano.shared(np.float32(0.01)), # on_epoch_finished=[AdjustVariable('update_learning_rate', start=0.02, stop=0.016)], max_epochs=18, eval_size=0.2, verbose=1, ) net0.fit(train_x, train_y) pred = net0.predict_proba(test_x).astype(np.float32) return pred
layers0 = [('input', InputLayer), ('dense0', DenseLayer), ('dropout0', DropoutLayer), ('dense1', DenseLayer), ('output', DenseLayer)] net0 = NeuralNet(layers=layers0, input_shape=(None, num_features), dense0_num_units=100, dropout0_p=0.1, dense1_num_units=100, output_num_units=num_classes, output_nonlinearity=softmax, update=nesterov_momentum, update_learning_rate=0.3, update_momentum=0.8, #objective_loss_function = binary_crossentropy, train_split=TrainSplit(0.1), verbose=1, max_epochs=20) net0.fit(X, y) print('Prediction Complete') preds = net0.predict_proba(X_test)[:, 1] submission = pd.DataFrame(preds, index=ids, columns=['target']) submission.to_csv('Neural_Network.csv')
predictions = [] gbm = xgb.train(params,xgb.DMatrix(result.iloc[train_index,:],y.iloc[train_index]),num_trees) rf.fit(X_train[train_index,:],y[train_index]) ada.fit(X_train[train_index,:],y[train_index]) svr.fit(X_train[train_index,:],y[train_index]) knn.fit(X_train[train_index,:],y[train_index]) y_train = encoder.fit_transform(y[train_index]).astype(np.int32) net0.fit(X_train[train_index,:],y_train) predictions.append(svr.predict_proba(X_train[test_index])[:,1]) predictions.append(gbm.predict(xgb.DMatrix(result.iloc[test_index]))) predictions.append(ada.predict_proba(X_train[test_index])[:,1]) predictions.append(rf.predict_proba(X_train[test_index])[:,1]) predictions.append(knn.predict_proba(X_train[test_index])[:,1]) predictions.append(net0.predict_proba(X_train[test_index])[:,1]) netAccuracy.append(roc_auc_score(y.iloc[test_index],net0.predict_proba(X_train[test_index])[:,1])) starting_values = [1./6]*len(predictions) # train_eval_probs = 0.30*svr.predict_proba(X_train[test_index])[:,1] + 0.15*gbm.predict(xgb.DMatrix(result.iloc[test_index])) \ # + 0.15*ada.predict_proba(X_train[test_index])[:,1] + 0.35*rf.predict_proba(X_train[test_index])[:,1] \ # + 0.05*knn.predict_proba(X_train[test_index])[:,1] res = minimize(log_loss_func, starting_values, method='SLSQP', bounds=bounds, constraints=cons) print('Ensamble Score: {best_score}'.format(best_score=res['fun'])) print('Best Weights: {weights}'.format(weights=res['x'])) weights.append(res['x']) # aucList.append(roc_auc_score(y.iloc[test_index],train_eval_probs)) print "Deep Learning - CNN" print sum(netAccuracy)/10.
def neuralNetwork(x_train, y_train, x_test, y_test): global get_test #use neural network to classifier print "Neural Network Model Learning..." num_train_case = x_train.shape[0] #get random permutation index index = np.random.permutation(num_train_case) #random permute x_train and y_train x_train = x_train[index, :] y_train = y_train[index] #encode y encoder = LabelEncoder() y_train = encoder.fit_transform(y_train).astype(np.int32) #print encoder.classes_ #scale x_train #scaler = StandardScaler() #x_train = scaler.fit_transform(x_train) #scale x_test #x_test = scaler.transform(x_test.astype(np.float32)) num_classes = len(encoder.classes_) num_features = x_train.shape[1] #Train Neural Net layers0 = [('input', InputLayer), ('dense0', DenseLayer), ('dropout', DropoutLayer), ('dense1', DenseLayer), ('output', DenseLayer)] net0 = NeuralNet(layers=layers0, input_shape=(None, num_features), dense0_num_units=2000, dropout_p=0.5, dense1_num_units=200, output_num_units=num_classes, output_nonlinearity=softmax, update=nesterov_momentum, update_learning_rate=0.01, update_momentum=0.9, eval_size=0.2, verbose=1, max_epochs=20) ''' epochs = [] def on_epoch_finished(nn, train_history): epochs[:] = train_history if len(epochs) > 1: raise StopIteration() net0 = NeuralNet( layers=[ ('input', InputLayer), ('hidden1', DenseLayer), ('dropout1', DropoutLayer), ('hidden2', DenseLayer), #('dropout2', DropoutLayer), ('output', DenseLayer), ], input_shape=(None, num_features), output_num_units=num_classes, output_nonlinearity=softmax, hidden1_num_units=200, hidden2_num_units=200, dropout_p=0.5, update=nesterov_momentum, update_learning_rate=0.01, update_momentum=0.9, max_epochs=20, on_epoch_finished=on_epoch_finished, ) #net0 = clone(nn_def) ''' #model learning net0.fit(x_train, y_train) #probability prediction y_prob = net0.predict_proba(x_test) if get_test == True: #the data is from real test set #output to file output_result(y_prob) else: #the test set is split from the train set, compute the loss function value #encode string label 'Class_1', 'Class_2',... to [0,1,...,8] y_true = encoder.transform(y_test).astype(np.int32) #compute the value for loss function score = logloss_mc(y_true, y_prob) print( " -- Multiclass logloss on validation set: {:.5f}.".format(score))
weights.append(res['x']) # aucList.append(roc_auc_score(y.iloc[test_index],train_eval_probs)) print "Deep Learning - CNN" print sum(netAccuracy)/10. #%% resultNet = net0.predict_proba(X_test) result = gbm.predict(xgb.DMatrix(X_test)) result_lb = lb.predict_proba(X_test) submit = pd.read_csv('/Users/weizhi/Desktop/kaggle walmart competetion/sample_submission.csv') Id = submit['VisitNumber'] submit.iloc[:,1:] = resultNet submit.iloc[:,0] = Id submit.to_csv('/Users/weizhi/Desktop/kaggle walmart competetion/deeplearning.csv',index = False)
}), (layers.DropoutLayer, { 'p': 0.5 }), (layers.DenseLayer, { 'num_units': 10, 'nonlinearity': nonlinearities.softmax })] net = NeuralNet( layers=layer, update=updates.adam, update_learning_rate=theano.shared(float32(0.002)), verbose=1, batch_iterator_train=batch_iterator_train, on_epoch_finished=[ AdjustVariable('update_learning_rate', start=0.002, stop=0.0005) ], eval_size=0.0, max_epochs=30, ) print("Training neural network...") net.fit(X, y) del X X_test, ids = load_test_data(path, grayscale=False, img_shape=IMG_SHAPE) print("Predicting on test data...") y_proba = net.predict_proba(X_test) make_submission('../output/submission_01.csv', y_proba, ids)
# This is silly, but we don't want a stratified K-Fold here # To compensate we need to pass in the y_tensor_type and the loss. regression=True, y_tensor_type=T.imatrix, objective_loss_function=binary_crossentropy, max_epochs=50, eval_size=None, verbose=1, ) X_train, y_train = shuffle(X_train, y_train, random_state=888) net.fit(X_train, y_train) y_pred = net.predict_proba(X_test)[:, 0] score = metrics.roc_auc_score(y_test_pd, y_pred) scores.append(score) total_pred = np.concatenate((total_pred, y_pred)) total_test = np.concatenate((total_test, y_test_pd)) print("Global ROC score", metrics.roc_auc_score(total_test, total_pred)) print(scores) print(np.array(scores).mean()) else: clf.fit(X, train.WnvPresent)
# AdjustVariable('update_learning_rate', start=0.01, stop=0.0001), # AdjustVariable('update_momentum', start=0.9, stop=0.999), # ], #on_epoch_finished=[ # AdjustVariable(learning_rate, target=0, half_life=4), #], eval_size=0.2, verbose=1, max_epochs=32, batch_iterator_train=BatchIterator(batch_size=64), batch_iterator_test=BatchIterator(batch_size=64)) net0.fit(X, y) X_train, X_valid, y_train, y_valid = net0.train_test_split( X, y, net0.eval_size) probas = net0.predict_proba(X_valid)[:, 0] #probas.loc[(probas[0]<0.3).values,[0]] = 0.1 print("ROC score", metrics.roc_auc_score(y_valid, (probas))) #yp = DataFrame(net0.predict_proba(X_test),columns=[ u'Class_1', u'Class_2', u'Class_3', u'Class_4', u'Class_5', u'Class_6', u'Class_7', u'Class_8', u'Class_9']) pred_prob = pd.DataFrame(net0.predict(X_test)) print pred_prob sample = pd.read_csv('/Users/IkkiTanaka/Documents/KDDCup/sampleSubmission.csv', header=None) preds = pd.concat([sample[0], pd.DataFrame(pred_prob)], axis=1) preds.to_csv('/Users/IkkiTanaka/Documents/KDDCup/pred/xgb/predNN1.csv', header=None, index=False) for i in xrange(0, 50): layers0 = [
# Decay the learning rate on_epoch_finished=[ AdjustVariable('update_learning_rate', start=0.01, stop=0.0001), AdjustVariable('update_momentum', start=0.9, stop=0.99), ], regression=True, y_tensor_type=T.imatrix, objective_loss_function=binary_crossentropy, #batch_iterator_train = BatchIterator(batch_size = 256), max_epochs=20, eval_size=0.1, #train_split =0.0, verbose=2, ) seednumber = 1235 np.random.seed(seednumber) net.fit(train, labels) preds = net.predict_proba(test)[:, 0] submission = pd.read_csv('../input/sample_submission.csv') submission["PredictedProb"] = preds submission.to_csv('NNbench.csv', index=False)
class NNnolearn: packageName = "com.brodagroup.machinelearning.classifer.NNnolearn" logger = None classifier = None # Initializer def __init__( self, num_classes=None, num_features=None, dense0_num_units=10, dropout_p=0.1, dense1_num_units=10, update_learning_rate=0.1, update_momentum=0.1, eval_size=0.1, max_epochs=10, verbose=5, ): self.logger = Logger(self.packageName).getLogger() self.logger.debug("Starting...") layers0 = [ ("input", InputLayer), ("dense0", DenseLayer), ("dropout1", DropoutLayer), ("dense1", DenseLayer), ("output", DenseLayer), ] self.classifier = NeuralNet( layers=layers0, input_shape=(None, num_features), dense0_num_units=dense0_num_units, dropout1_p=dropout_p, dense1_num_units=dense1_num_units, output_num_units=num_classes, output_nonlinearity=softmax, update=nesterov_momentum, update_learning_rate=update_learning_rate, update_momentum=update_momentum, eval_size=eval_size, max_epochs=max_epochs, verbose=verbose, ) self.num_classes = num_classes self.num_features = num_features self.dense0_num_units = dense0_num_units self.dropout_p = dropout_p self.dense1_num_units = dense1_num_units self.update_learning_rate = update_learning_rate self.update_momentum = update_momentum self.eval_size = eval_size self.max_epochs = max_epochs self.verbose = verbose return def __str__(self): return x def fit(self, X, y): self.classifier.fit(X, y) def predict(self, X): y_pred = self.classifier.predict(X) return y_pred def predict_proba(self, X): y_pred = self.classifier.predict_proba(X) return y_pred def get_params(self, deep=True): return { "num_classes": self.num_classes, "num_features": self.num_features, "dense0_num_units": self.dense0_num_units, "dropout_p": self.dropout_p, "dense1_num_units": self.dense1_num_units, "update_learning_rate": self.update_learning_rate, "eval_size": self.eval_size, "max_epochs": self.max_epochs, "verbose": self.verbose, } def set_params(self, **parameters): for parameter, value in parameters.items(): setattr(self, parameter, value) return self
objective_loss_function=categorical_crossentropy, train_split=TrainSplit(eval_size=0), verbose=1, max_epochs=1) # ==== Print out input shape for diagnosis ==== print(X.shape) print(training_target.shape) # ==== Train it for n iterations and validate on each iteration ==== spl = 80000 for i in range(epochs): net1.fit(x2[:spl, :], training_target[:spl]) print(i + 1) pred = net1.predict_proba(x2[spl:, :])[:, 1] val_auc[i] = roc_auc_score(training_target[spl:], pred) print(i + 1, "\t", round(val_auc[i] * 100, 3), "\t", round(max(val_auc) * 100, 3), "\t") pred_d = pd.read_csv(path_pred) del pred_d["t_id"] for f in features: for g in features: if f != g: if not (str(g) + "_" + str(f)) in pred_d.columns: pred_d[str(f) + "_" + str(g)] = pred_d[f] * pred_d[g] X = pred_d.values.astype(np.float32) X = scaler.transform(X).astype(np.float32) proba = net1.predict_proba(y2)[:, 1]
dropout_p=0.5, dense1_num_units=200, output_num_units=num_classes, output_nonlinearity=softmax, update=nesterov_momentum, update_learning_rate=0.01, update_momentum=0.9, eval_size=0.2, verbose=1, max_epochs=200) print "fitting nn model.." net0.fit(X_train, y_train) print "predicting probabilities using nn model..." proba = net0.predict_proba(X_test) ll.append(calc_ll_from_proba(proba, y_test)) print metrics.confusion_matrix( y_test.astype(int), np.argmax(proba, axis=1).astype(int)) print "logloss: ", ll[ncv] print "saving nn model..." net0.save_weights_to('weights/nn_%d.pkl' % ncv) net0 = None ncv += 1 ll = np.array(ll) print "logloss: ", ll
dropout3_p=0.3, hidden4_num_units=230, hidden4_nonlinearity=very_leaky_rectify, dropout4_p=0.5, hidden5_num_units=150, dropout5_p=0.35, output_num_units=2, output_nonlinearity=softmax, update_learning_rate=theano.shared(float32(0.02)), update_momentum=theano.shared(float32(0.9)), # regression=True, batch_iterator_train=FlipBatchIterator(batch_size=128), on_epoch_finished=[ AdjustVariable('update_learning_rate', start=0.02, stop=0.0001), AdjustVariable('update_momentum', start=0.9, stop=0.999), EarlyStopping(patience=80), ], max_epochs=2000, verbose=1, train_split=TrainSplit(eval_size=0.2), ) if __name__ == '__main__': X, y , Xtest= load2d() # load 2-d data net.fit(X.astype(theano.config.floatX), y.astype(np.int32).reshape(-1,)) ypred = net.predict_proba(Xtest.astype(theano.config.floatX)) with open('net3.res.pickle', 'wb') as f: pickle.dump(ypred, f) with open('net3.pickle', 'wb') as f: pickle.dump(net,f)
objective_loss_function=objectives.categorical_crossentropy, #objective=objectives.categorical_crossentropy, batch_iterator_train=BatchIterator(batch_size=batchSize), batch_iterator_test=BatchIterator(batch_size=batchSize), train_split=TrainSplit(eval_size=0.2, stratify=False), use_label_encoder=True, #use_label_encoder = False, regression=False, max_epochs=numberEpochs, verbose=1) #x_fit, x_eval, y_fit, y_eval= cross_validation.train_test_split(xTrain, y, test_size=0.2) net.fit(xTrain, y) predictY = net.predict_proba(xTrain) print(metrics.log_loss(originalY, predictY)) files = [f for f in listdir(testDir) if path.isfile(path.join(testDir, f))] xTest = np.zeros((len(files), imageSize), dtype='float32') index = 0 for fName in files: fileName = path.join(testDir, fName) im = Image.open(fileName) xTest[index, :] = np.reshape(im, imageSize) im.close() index += 1 xTest /= 255 xTest = xTest.reshape(xTest.shape[0], 1, imageShape[0], imageShape[1]).astype('float32')
update_learning_rate=theano.shared(np.float32(0.013)), ### regression=False, max_epochs=2000, train_split=TrainSplit(eval_size=0.1), #custom_score=('auc', lambda y_true, y_proba: roc_auc_score(y_true, y_proba[:, 1])), on_epoch_finished=[ AdjustVariable('update_learning_rate', start=0.013, stop=0.001), #AdjustVariable('update_momentum', start=0.9, stop=0.999), EarlyStopping(patience=60), ], verbose=1) clf.fit(X_train, y_train) from sklearn import metrics y_pred = clf.predict_proba(X_val0)[:, 1] score = metrics.roc_auc_score(y_val0, y_pred) print('score on extra set:%s' % score) model = 'Nolearn' # # predict on test set submission = '%s_score_%03f.csv' % (model, score) # create submission file predictions = clf.predict_proba(X_test)[:, 1] preds = pd.DataFrame({"patient_id": id_test, 'predict_screener': predictions}) test_ex_file = ('../input/test_patients_to_exclude.csv.gz') test_ex = pd.read_csv(test_ex_file, low_memory=False)
random_state=1) print("\n\nTraining") net = NeuralNet( layers=[ # three layers: one hidden layer ('input', layers.InputLayer), ('hidden', layers.DenseLayer), ('output', layers.DenseLayer), ], input_shape=(None, 6), hidden_num_units=5, output_nonlinearity=sigmoid, output_num_units=2, # optimization method: update=sgd, update_learning_rate=0.01, #update_momentum=0.9, regression=False, max_epochs=50, verbose=1, ) net.fit(X, y) pred = net.predict_proba(tX)[:,0] print("\t", log_loss(ty, pred)) print("\t", roc_auc_score(ty, pred))
class NNet(BaseEstimator, ClassifierMixin): def __init__( self, name='nameless_net', # used for saving, so maybe make it unique dense1_size=60, dense1_nonlinearity='tanh', dense1_init='orthogonal', dense2_size=None, dense2_nonlinearity=None, # inherits dense1 dense2_init=None, # inherits dense1 dense3_size=None, dense3_nonlinearity=None, # inherits dense2 dense3_init=None, # inherits dense2 learning_rate=0.001, learning_rate_scaling=100, momentum=0.9, momentum_scaling=100, max_epochs=3000, epoch_steps=None, dropout0_rate=0, # this is the input layer dropout1_rate=None, dropout2_rate=None, # inherits dropout1_rate dropout3_rate=None, # inherits dropout2_rate weight_decay=0, adaptive_weight_decay=False, batch_size=128, output_nonlinearity='softmax', auto_stopping=True, save_snapshots_stepsize=None, ): """ Create the network with the selected parameters. :param name: Name for save files :param dense1_size: Number of neurons for first hidden layer :param dense1_nonlinearity: The activation function for the first hidden layer :param dense1_init: The weight initialization for the first hidden layer :param learning_rate: The (initial) learning rate (how fast the network learns) :param learning_rate_scaling: The total factor to gradually decrease the learning rate by :param momentum: The (initial) momentum :param momentum_scaling: Similar to learning_rate_scaling :param max_epochs: Total number of epochs (at most) :param dropout1_rate: Percentage of connections dropped each step for first hidden layer :param weight_decay: Palatalizes the weights by L2 norm (regularizes but decreases results) :param adaptive_weight_decay: Should the weight decay adapt automatically? :param batch_size: How many samples to send through the network at a time :param auto_stopping: Stop early if the network seems to stop performing well :param pretrain: Filepath of the previous weights to start at (or None) :return: """ """ Input argument storage: automatically store all locals, which should be exactly the arguments at this point, but storing a little too much is not a big problem. """ params = locals() del params['self'] #self.__dict__.update(params) self.parameter_names = sorted(params.keys()) """ Check the parameters and update some defaults (will be done for 'self', no need to store again). """ self.set_params(**params) def init_net(self, feature_count, class_count=NCLASSES, verbosity=VERBOSITY >= 2): """ Initialize the network (needs to be done when data is available in order to set dimensions). """ if VERBOSITY >= 1: print 'initializing network {0:s} {1:d}x{2:d}x{3:d}'.format( self.name, self.dense1_size or 0, self.dense2_size or 0, self.dense3_size or 0) if VERBOSITY >= 2: print 'parameters: ' + ', '.join( '{0:s} = {1:}'.format(k, v) for k, v in self.get_params(deep=False).items()) self.feature_count = feature_count self.class_count = class_count """ Create the layers and their settings. """ self.layers = [ ('input', InputLayer), ] self.params = { 'dense1_num_units': self.dense1_size, 'dense1_nonlinearity': nonlinearities[self.dense1_nonlinearity], 'dense1_W': initializers[self.dense1_init], 'dense1_b': Constant(0.), } if self.dropout0_rate: self.layers += [('dropout0', DropoutLayer)] self.params['dropout0_p'] = self.dropout0_rate self.layers += [ ('dense1', DenseLayer), ] if self.dropout1_rate: self.layers += [('dropout1', DropoutLayer)] self.params['dropout1_p'] = self.dropout1_rate if self.dense2_size: self.layers += [('dense2', DenseLayer)] self.params.update({ 'dense2_num_units': self.dense2_size, 'dense2_nonlinearity': nonlinearities[self.dense2_nonlinearity], 'dense2_W': initializers[self.dense2_init], 'dense2_b': Constant(0.), }) else: assert not self.dense3_size, 'There cannot be a third dense layer without a second one' if self.dropout2_rate: assert self.dense2_size is not None, 'There cannot be a second dropout layer without a second dense layer.' self.layers += [('dropout2', DropoutLayer)] self.params['dropout2_p'] = self.dropout2_rate if self.dense3_size: self.layers += [('dense3', DenseLayer)] self.params.update({ 'dense3_num_units': self.dense3_size, 'dense3_nonlinearity': nonlinearities[self.dense3_nonlinearity], 'dense3_W': initializers[self.dense3_init], 'dense3_b': Constant(0.), }) if self.dropout3_rate: assert self.dense2_size is not None, 'There cannot be a third dropout layer without a third dense layer.' self.layers += [('dropout3', DropoutLayer)] self.params['dropout3_p'] = self.dropout2_rate self.layers += [('output', DenseLayer)] self.params.update({ 'output_nonlinearity': nonlinearities[self.output_nonlinearity], 'output_W': GlorotUniform(), 'output_b': Constant(0.), }) """ Create meta parameters and special handlers. """ if VERBOSITY >= 3: print 'learning rate: {0:.6f} -> {1:.6f}'.format( abs(self.learning_rate), abs(self.learning_rate) / float(self.learning_rate_scaling)) print 'momentum: {0:.6f} -> {1:.6f}'.format( abs(self.momentum), 1 - ((1 - abs(self.momentum)) / float(self.momentum_scaling))) self.step_handlers = [ LinearVariable('update_learning_rate', start=abs(self.learning_rate), stop=abs(self.learning_rate) / float(self.learning_rate_scaling)), LinearVariable( 'update_momentum', start=abs(self.momentum), stop=1 - ((1 - abs(self.momentum)) / float(self.momentum_scaling))), StopNaN(), ] self.end_handlers = [ SnapshotEndSaver(base_name=self.name), TrainProgressPlotter(base_name=self.name), ] snapshot_name = 'nn_' + params_name(self.params, prefix=self.name)[0] if self.save_snapshots_stepsize: self.step_handlers += [ SnapshotStepSaver(every=self.save_snapshots_stepsize, base_name=snapshot_name), ] if self.auto_stopping: self.step_handlers += [ StopWhenOverfitting(loss_fraction=0.9, base_name=snapshot_name), StopAfterMinimum(patience=40, base_name=self.name), ] weight_decay = shared(float32(abs(self.weight_decay)), 'weight_decay') if self.adaptive_weight_decay: self.step_handlers += [ AdaptiveWeightDecay(weight_decay), ] if self.epoch_steps: self.step_handlers += [ BreakEveryN(self.epoch_steps), ] """ Create the actual nolearn network with information from __init__. """ self.net = NeuralNet( layers=self.layers, objective=partial(WeightDecayObjective, weight_decay=weight_decay), input_shape=(None, feature_count), output_num_units=class_count, update=nesterov_momentum, # todo: make parameter update_learning_rate=shared(float32(self.learning_rate)), update_momentum=shared(float(self.weight_decay)), on_epoch_finished=self.step_handlers, on_training_finished=self.end_handlers, regression=False, max_epochs=self.max_epochs, verbose=verbosity, batch_iterator_train=BatchIterator(batch_size=self.batch_size), batch_iterator_test=BatchIterator(batch_size=self.batch_size), eval_size=0.1, #custom_score = ('custom_loss', categorical_crossentropy), **self.params) self.net.parent = self self.net.initialize() return self.net def get_params(self, deep=True): return OrderedDict( (name, getattr(self, name)) for name in self.parameter_names) def set_params(self, **params): """ Set all the parameters. """ for name, val in params.items(): assert name in self.parameter_names, '"{0:s}" is not a valid parameter name (known parameters: "{1:s}")'.format( name, '", "'.join(self.parameter_names)) setattr(self, name, val) """ Arguments checks. """ assert self.dropout1_rate is None or 0 <= self.dropout1_rate < 1, 'Dropout rate 1 should be a value between 0 and 1 (value: {0})'.format( self.dropout1_rate) assert self.dropout2_rate is None or 0 <= self.dropout2_rate < 1, 'Dropout rate 2 should be a value between 0 and 1, or None for inheritance (value: {0})'.format( self.dropout2_rate) assert self.dropout3_rate is None or 0 <= self.dropout3_rate < 1, 'Dropout rate 3 should be a value between 0 and 1, or None for inheritance (value: {0})'.format( self.dropout3_rate) assert self.dense1_nonlinearity in nonlinearities.keys( ), 'Linearity 1 should be one of "{0}", got "{1}" instead.'.format( '", "'.join(nonlinearities.keys()), self.dense1_nonlinearity) assert self.dense2_nonlinearity in nonlinearities.keys() + [ None ], 'Linearity 2 should be one of "{0}", got "{1}" instead.'.format( '", "'.join(nonlinearities.keys()), self.dense2_nonlinearity) assert self.dense3_nonlinearity in nonlinearities.keys() + [ None ], 'Linearity 3 should be one of "{0}", got "{1}" instead.'.format( '", "'.join(nonlinearities.keys()), self.dense3_nonlinearity) assert self.dense1_init in initializers.keys( ), 'Initializer 1 should be one of "{0}", got "{1}" instead.'.format( '", "'.join(initializers.keys()), self.dense1_init) assert self.dense2_init in initializers.keys() + [ None ], 'Initializer 2 should be one of "{0}", got "{1}" instead.'.format( '", "'.join(initializers.keys()), self.dense2_init) assert self.dense3_init in initializers.keys() + [ None ], 'Initializer 3 should be one of "{0}", got "{1}" instead.'.format( '", "'.join(initializers.keys()), self.dense3_init) """ Argument defaults. """ if self.dense2_nonlinearity is None: self.dense2_nonlinearity = self.dense1_nonlinearity if self.dense2_init is None: self.dense2_init = self.dense1_init if self.dense3_nonlinearity is None: self.dense3_nonlinearity = self.dense2_nonlinearity if self.dense3_init is None: self.dense3_init = self.dense2_init if self.dropout2_rate is None and self.dense2_size: self.dropout2_rate = self.dropout1_rate if self.dropout3_rate is None and self.dense3_size: self.dropout3_rate = self.dropout2_rate def fit(self, X, y, random_sleep=None): if random_sleep: sleep(random_sleep * random()) # this is to prevent compiler lock problems labels = y - y.min() #todo: don't use labels.max(), occasionally (rarely) it will not have the highest class self.init_net(feature_count=X.shape[1], class_count=labels.max() + 1) net = self.net.fit(X, labels) self.save() return net def interrupted_fit(self, X, y): """ DEPRECATED """ labels = y - y.min() self.init_net(feature_count=X.shape[1], class_count=labels.max() + 1) knowledge = get_knowledge(self.net) for epoch in range(0, self.max_epochs, self.epoch_steps): set_knowledge(self.net, knowledge) self.init_net(feature_count=X.shape[1], class_count=labels.max() + 1) print 'epoch {0:d}: learning {1:d} epochs'.format( epoch, self.epoch_steps) self.net.fit(X, labels) ratio = mean([d['valid_loss'] for d in self.net._train_history[-self.epoch_steps:]]) / \ mean([d['train_loss'] for d in self.net._train_history[-self.epoch_steps:]]) if ratio < 0.85: self.weight_decay *= 1.3 if ratio > 0.95: self.weight_decay /= 1.2 self.init_net(feature_count=X.shape[1], class_count=labels.max() + 1) knowledge = get_knowledge(self.net) exit() net = self.net.fit(X, labels) self.save() return net def predict_proba(self, X): probs = self.net.predict_proba(X) if not isfinite(probs).sum(): errmsg = 'network "{0:s}" predicted infinite/NaN probabilities'.format( self.name) stderr.write(errmsg) raise DivergenceError(errmsg) return probs def predict(self, X): return self.net.predict(X) def score(self, X, y, **kwargs): return self.net.score(X, y) def save(self, filepath=None): assert hasattr( self, 'net' ), 'Cannot save a network that is not initialized; .fit(X, y) something first [or use net.initialize(..) for random initialization].' parameters = self.get_params(deep=False) filepath = filepath or join(NNET_STATE_DIR, self.name) if VERBOSITY >= 1: print 'saving network to "{0:s}.net.npz|json"'.format(filepath) with open(filepath + '.net.json', 'w+') as fh: dump([parameters, self.feature_count, self.class_count], fp=fh, indent=2) save_knowledge(self.net, filepath + '.net.npz') @classmethod def load(cls, filepath=None, name=None): """ :param filepath: The base path (without extension) to load the file from, OR: :param name: The name of the network to load (if filename is not given) :return: The loaded network """ filepath = filepath or join(NNET_STATE_DIR, name) if VERBOSITY >= 1: print 'loading network from "{0:s}.net.npz|json"'.format(filepath) with open(filepath + '.net.json', 'r') as fh: [parameters, feature_count, class_count] = load(fp=fh) nnet = cls(**parameters) nnet.init_net(feature_count=feature_count, class_count=class_count) load_knowledge(nnet.net, filepath + '.net.npz') return nnet
AdjustVariable('update_momentum', start=0.9, stop=0.999), ], regression=False, # flag to indicate we're dealing with regression problem max_epochs=500, # we want to train this many epochs verbose=1 ) clf.fit(x_train,y_train) # if 1: # y_pred = clf.predict_proba(x_test) # # filename = 'testdata_aug_d' # savefile = open(filename+'.pkl', 'wb') # cPickle.dump((x_test, y_pred, name1),savefile,-1) # savefile.close() if 1: y_pred = clf.predict(x_test) print "Accuracy:", zero_one_loss(y_test, y_pred) print "Classification report:" print classification_report(y_test, y_pred) print 'Confusion matrix:' print confusion_matrix(y_test,y_pred) else: x_test = np.asarray(x_test,dtype=np.float32) ypred = clf.predict_proba(x_test) y_str = ['Class_1','Class_2','Class_3','Class_4','Class_5','Class_6','Class_7','Class_8','Class_9'] kcsv.print_csv(ypred, name1, y_str,indexname='id')
def do_nn(xTrain, yTrain, test_x=None, test_y=None, dropout_in=0.2, dense0_num=600, dropout_p=0.4, dense1_num=1200, update_learning_rate=0.00002, update_momentum=0.9, test_ratio=0.2, max_epochs=40, search=False): num_features = len(xTrain[0, :]) num_classes = 2 print num_features if search == False: layers0 = [('input', InputLayer), ('dropoutin', DropoutLayer), ('dense0', DenseLayer), ('dropout', DropoutLayer), ('dense1', DenseLayer), ('output', DenseLayer)] clf = NeuralNet(layers=layers0, input_shape=(None, num_features), dropoutin_p=dropout_in, dense0_num_units=dense0_num, dropout_p=dropout_p, dense1_num_units=dense1_num, output_num_units=num_classes, output_nonlinearity=softmax, update=nesterov_momentum, update_learning_rate=update_learning_rate, update_momentum=update_momentum, eval_size=test_ratio, verbose=1, max_epochs=max_epochs) clf.fit(xTrain, yTrain) if test_x != None and test_y != None: probas_nn = clf.predict_proba(test_x)[:, 1] score_nn = roc_auc_score(test_y, probas_nn) print("NN ROC score", score_nn) return clf else: dropout_in_list = [0.2] dense0_num_list = [500, 1000, 1500] dropout_p_list = [0.5] dense1_num_list = [400, 800, 1200] info = {} for d_in in dropout_in_list: for d_01 in dropout_p_list: for d0 in dense0_num_list: for d1 in dense1_num_list: print 'dropout_in = ', d_in print 'dense0_num = ', d0 print 'dropout_p = ', d_01 print 'dense0_num = ', d1 layers0 = [('input', InputLayer), ('dropoutin', DropoutLayer), ('dense0', DenseLayer), ('dropout', DropoutLayer), ('dense1', DenseLayer), ('output', DenseLayer)] clf = NeuralNet(layers=layers0, input_shape=(None, num_features), dropoutin_p=dropout_in, dense0_num_units=dense0_num, dropout_p=dropout_p, dense1_num_units=dense1_num, output_num_units=num_classes, output_nonlinearity=softmax, update=nesterov_momentum, update_learning_rate=update_learning_rate, update_momentum=update_momentum, eval_size=test_ratio, verbose=1, max_epochs=max_epochs) clf.fit(xTrain, yTrain) probas_nn = clf.predict_proba(test_x)[:, 1] score_nn = roc_auc_score(test_y, probas_nn) print("NN ROC score", score_nn) info[d_in, d0, d_01, d1] = score_nn for md in info: scores = info[md] print('NN dropout_in = %.5f, dense0_num = %d, dropout_p = %.5f, dense1_num = %d, ROC score = %.5f(%.5f)' % \ (md[0], md[1], md[2], md[3], scores.mean(), scores.std()))
def make_predictions(train, target, test, ranking=False, load_list=[]): result = np.zeros(len(test)) seed_list = [1234, 2345, 3456, 4567, 5678, 6789, 7890, 8901, 9012, 123] print("Training model1...") if "linear" not in load_list: model1 = sklearn.linear_model.SGDClassifier(loss="log", alpha=0.01, l1_ratio=0, n_iter=100) model1.fit(train, target) pickle.dump(model1, open("final_models/linear/lr.pkl", "wb")) else: model1 = pickle.load(open("final_models/linear/lr.pkl", "rb")) pred1 = model1.predict_proba(test)[:, 1] print("Training model2...") pred2 = np.zeros(len(pred1)) for i in range(10): if "xgb" not in load_list: model2 = xgb.XGBClassifier(n_estimators=500, max_depth=4, colsample_bytree=0.6, subsample=0.8, learning_rate=0.09, seed=seed_list[i]) model2.fit(train, target) pickle.dump(model2, open("final_models/xgb/xgb_n_"+str(i)+".pkl", "wb")) else: model2 = pickle.load(open("final_models/xgb/xgb_n_"+str(i)+".pkl", "rb")) pred2 += model2.predict_proba(test)[:, 1] pred2 /= 10 print("Training model3...") pred3 = np.zeros(len(pred2)) for i in range(10): if "nn" not in load_list: np.random.seed(seed_list[i]) num_classes = 2 layers0 = [('input', InputLayer), ('dense0', DenseLayer), ('dropout1', DropoutLayer), ('dense1', DenseLayer), ('dropout2', DropoutLayer), ('dense2', DenseLayer), ('output', DenseLayer)] model3 = NeuralNet(layers=layers0, input_shape=(None, train.shape[1]), dense0_num_units=150, dropout1_p=0.4, dense1_num_units=150, dropout2_p=0.4, dense2_num_units=150, output_num_units=num_classes, output_nonlinearity=softmax, update=nesterov_momentum, update_learning_rate=0.001, update_momentum=0.9, eval_size=0.01, verbose=0, max_epochs=100, use_label_encoder=True) model3.fit(train, target) pickle.dump(model3, open("final_models/nn/nn_n_"+str(i)+".pkl", "wb")) else: model3 = pickle.load(open("final_models/nn/nn_n_"+str(i)+".pkl", "rb")) pred3 += model3.predict_proba(test)[:, 1] pred3 /= 10 if ranking: pred1 = scipy.stats.rankdata(pred1) pred2 = scipy.stats.rankdata(pred2) pred3 = scipy.stats.rankdata(pred3) result = 0.21*pred1 + 0.47*pred2 + 0.32*pred3 return result
def train(): weather = load_weather() training = load_training() X = assemble_X(training, weather) mean, std = normalize(X) y = assemble_y(training) col_labels = ["date.year","date.month","date.day","block","lat","long",] for obs in ["Tmax","Tmin","Tavg","DewPoint","WetBulb","PrecipTotal","Depart","Sunrise","Sunset","Heat","Cool","ResultSpeed","ResultDir"]: for day in ["1","2","3","5","8","12"]: col_labels.append(obs+"_"+day) for i_spec in range(6): col_labels.append("species_"+str(i_spec)) X_file = csv.writer(open("X.csv", "w")) X_file.writerow(col_labels) for row in X: X_file.writerow(row) y_file = csv.writer(open("y.csv", "w")) y_file.writerow(["y"]) for row in y: y_file.writerow(row) input_size = len(X[0]) learning_rate = theano.shared(np.float32(0.1)) net = NeuralNet( layers=[ ('input', InputLayer), ('hidden1', DenseLayer), ('dropout1', DropoutLayer), ('hidden2', DenseLayer), ('dropout2', DropoutLayer), ('output', DenseLayer), ], # layer parameters: input_shape=(None, input_size), hidden1_num_units=600, dropout1_p=0.5, hidden2_num_units=400, dropout2_p=0.5, output_nonlinearity=sigmoid, output_num_units=1, # optimization method: update=nesterov_momentum, update_learning_rate=learning_rate, update_momentum=0.9, # Decay the learning rate on_epoch_finished=[ AdjustVariable(learning_rate, target=0, half_life=20), ], # This is silly, but we don't want a stratified K-Fold here # To compensate we need to pass in the y_tensor_type and the loss. regression=True, y_tensor_type = T.imatrix, objective_loss_function = binary_crossentropy, max_epochs=500, eval_size=0.1, verbose=1, ) X, y = shuffle(X, y, random_state=123) net.fit(X, y) _, X_valid, _, y_valid = net.train_test_split(X, y, net.eval_size) probas = net.predict_proba(X_valid)[:,0] print("ROC score", metrics.roc_auc_score(y_valid, probas)) return net, mean, std
# pre-processing data df.target = df.target.apply(lambda x: x.split("_", 1)[1]) # convert class_x -> x df.target = df.target.astype(np.int32) df.target = df.target - 1 df = df.iloc[:,1:] df = df.reindex(np.random.permutation(df.index)) df_test = df_test.iloc[:,1:] # take out id col X_test = df_test.values.astype(np.float32) X, y = df.values[:,:93].astype(np.float32), df.values[:,93].astype(np.int32) # training result = myNN.fit(X,y) # predict and prepare submission file y_prob = myNN.predict_proba(X_test) y_prob_df = pd.DataFrame(y_prob) id_df = pd.DataFrame({'id':range(1,len(df_test)+1)}) final_df = pd.concat([id_df, y_prob_df], axis=1) final_df.columns = ['id', 'Class_1', 'Class_2', 'Class_3', 'Class_4', 'Class_5', 'Class_6', 'Class_7', 'Class_8', 'Class_9'] final_df.to_csv('final_result.csv', index = False)
def main(): # my code here y_train=load_train() y_test =load_test() X_train=load_images("/home/pratik/Desktop/Skinzy Code SVM/Datasets2/scabies_train.gz") X_test=load_images("/home/pratik/Desktop/Skinzy Code SVM/Datasets2/scabies_test.gz") X_train,y_train= shuffle(X_train, y_train, random_state=0) X_val=X_train[80:] X_train=X_train[:80] y_val=y_train[80:] y_train=y_train[:80] y_train =np.array(y_train) y_test=np.array(y_test) y_val=np.array(y_val) y_test = y_test.astype(np.uint8) y_train = y_train.astype(np.uint8) y_val = y_val.astype(np.uint8) X_train,y_train= shuffle(X_train, y_train, random_state=0) #print("Plotting the graph") #plt.imshow(X_train[0][0]) net1 = NeuralNet( layers=[('input', layers.InputLayer), ('conv2d1', layers.Conv2DLayer), ('maxpool1', layers.MaxPool2DLayer), ('conv2d2', layers.Conv2DLayer), ('maxpool2', layers.MaxPool2DLayer), ('dropout1', layers.DropoutLayer), ('dense', layers.DenseLayer), ('dropout2', layers.DropoutLayer), ('output', layers.DenseLayer), ], # input layer input_shape=(None, 3, 64,64), # layer conv2d1 conv2d1_num_filters=32, conv2d1_filter_size=(5, 5), conv2d1_nonlinearity=lasagne.nonlinearities.rectify, conv2d1_W=lasagne.init.GlorotUniform(), # layer maxpool1 maxpool1_pool_size=(3, 3), maxpool1_stride=1, maxpool1_pad=0, # layer conv2d2 conv2d2_num_filters=32, conv2d2_filter_size=(5, 5), conv2d2_nonlinearity=lasagne.nonlinearities.rectify, # layer maxpool2 maxpool2_pool_size=(3, 3), maxpool2_stride=1, maxpool2_pad=0, # dropout1 dropout1_p=0.5, # dense dense_num_units=256, dense_nonlinearity=lasagne.nonlinearities.rectify, # dropout2 dropout2_p=0.5, # output output_nonlinearity=lasagne.nonlinearities.softmax, output_num_units=2, # optimization method params update=nesterov_momentum, update_learning_rate=0.001, update_momentum=0.9, max_epochs=50, verbose=1, ) print ("Training starts :") net1.fit(X_train, y_train) #preds = net1.predict(X_test[0]) out = X_train[0].reshape(-1, 3, 64, 64) pred=net1.predict_proba(out) print pred #cm = confusion_matrix(y_test, preds) #plt.matshow(cm) #plt.title('Confusion matrix') #plt.colorbar() #plt.ylabel('True label') #plt.xlabel('Predicted label') #plt.show() #print (net1.predict_proba(X_test)) sys.setrecursionlimit(9999999) joblib.dump(net1, 'AndroidFileUpload/classifier_2disease/cnn_1.pkl',compress=9) y_true, y_pred = y_test, net1.predict(X_test) # Get our predictions print(classification_report(y_true, y_pred)) # Classification on each digit print 'The accuracy is:', accuracy_score(y_true, y_pred) # visualize.plot_conv_weights(net1.layers_['conv2d1']) dense_layer = layers.get_output(net1.layers_['dense'], deterministic=True) output_layer = layers.get_output(net1.layers_['output'], deterministic=True) input_var = net1.layers_['input'].input_var f_output = theano.function([input_var], output_layer) f_dense = theano.function([input_var], dense_layer) instance = X_test[0][None, :, :] #%timeit -n 500 f_output(instance) train_features = f_dense(X_train) test_features=f_dense(X_test) train_labels=y_train test_labels=y_test ''' Logistic Regression clf2 = LogisticRegression().fit(train_features, train_labels) joblib.dump(clf2, 'AndroidFileUpload/classifier_2disease/log_reg.pkl',compress=9)''' #SVM clf = svm.SVC(kernel="linear",C=10000.0,probability=True) clf.fit(train_features,train_labels) sys.setrecursionlimit(9999999) joblib.dump(clf, 'AndroidFileUpload/classifier_2disease/svm.pkl',compress=9) pred=clf.predict(test_features) print(classification_report(test_labels, pred)) print 'The accuracy is:', accuracy_score(test_labels, pred) pred=clf.predict_proba(test_features) print pred '''#KNN
output_num_units=4, # optimization method: update=nesterov_momentum, update_learning_rate=learning_rate, update_momentum=0.899, # Decay the learning rate on_epoch_finished=[ AdjustVariable(learning_rate, target=0, half_life=4), ], # This is silly, but we don't want a stratified K-Fold here # To compensate we need to pass in the y_tensor_type and the loss. regression=True, y_tensor_type = T.imatrix, objective_loss_function = binary_crossentropy, max_epochs=75, eval_size=0.1, verbose=1, ) X, y = shuffle(Xtrh, y, random_state=123) net.fit(X, y) _, X_valid, _, y_valid = net.train_test_split(X, y, net.eval_size) probas = net.predict_proba(X_valid)[:,0] print("ROC score", metrics.roc_auc_score(y_valid, probas))
for dy in range(-6, 6): if nuclei_map[limit(i + dx, 0, SIZE - 1), limit(j + dy, 0, SIZE - 1)]: cover = True for xx, yy in coords: if i == xx and j == yy: print("Found coordinate of mitosis: ", (xx, yy)) print("Cover = ", cover) if cover: patches.append((i, j)) if len(patches) >= BUFFER: print("Evaluating!") patches2 = get_patches(patches) prob = nn.predict_proba(patches2) for k in range(0, len(patches)): sx, sy = patches[k] patch_probs[sx, sy] = prob[k, 1] for xx, yy in coords: if sx == xx and sy == yy: print("Found coordinate of mitosis: ", (xx, yy)) print("Prob = ", prob[k, 1]) patches = [] with suppress_stdout(): nn.load_params_from(netfile) if len(patches) >= 1: print("Evaluating!")
# In[2]: import pickle pickle.dump(net0, open('lasagne_model.pkl', 'wb')) # In[106]: print 'score', net0.score(features_scaled, labels) # In[111]: print 'logloss', log_loss(labels, net0.predict_proba(features_scaled)) # In[16]: test_features, _ = get_features(test) scaler.fit(test_features) test_features = scaler.transform(test_features) # In[18]: proba = net0.predict_proba(test_features) prediction = pd.DataFrame(proba, columns=label_encoder.classes_)
# ==== Neural network definition ==== net1 = NeuralNet(layers=l_out, update=adadelta, update_rho=0.95, update_learning_rate=1.0, objective_loss_function=categorical_crossentropy, train_split=TrainSplit(eval_size=0), verbose=0, max_epochs=1) # ==== Print out input shape for diagnosis ==== print(X.shape) print(training_target.shape) # ==== Train it for n iterations and validate on each iteration ==== for i in range(epochs): net1.fit(X, training_target) pred = net1.predict_proba(Y)[:, 1] val_auc[i] = roc_auc_score(validation_target, pred) print(i + 1, "\t", round(val_auc[i] * 100, 3), "\t", round(max(val_auc) * 100, 3), "\t") # ==== Make of the plot of the validation accuracy per iteration ==== pyplot.plot(val_auc, linewidth=2) pyplot.axhline(y=max(val_auc), xmin=0, xmax=epochs, c="blue", linewidth=3, zorder=0) pyplot.grid() pyplot.title("Maximum AUC is " + str(round(max(val_auc) * 100, 3)) + '%') pyplot.xlabel("Epoch") pyplot.ylabel("Validation AUC") pyplot.show()
print("Fitting Sample 0") net0.fit(X, y) for i in range(1, 14): print("Loading Sample " + str(i)) X, y, encoder, scaler1 = load_train_data( datapath + "train_fixed_data.csv", i) print("Fitting Sample " + str(i)) net0.fit(X, y) print("Fitting Complete") test_splits = 14 print("Loading Test Sample 0") X_test, ids = load_test_data(datapath + "test_fixed_data.csv", scaler, 0) print("Predicting Test Sample 0") preds = net0.predict_proba(X_test)[:, 1] submission = pd.DataFrame(preds, index=ids, columns=['target']) submission.to_csv('BTB_Lasagne.csv') for i in range(1, test_splits + 1): print("Loading Test Sample " + str(i)) X_test, ids = load_test_data(datapath + "test_fixed_data.csv", scaler, i) print("Predicting Test Sample " + str(i)) preds = net0.predict_proba(X_test)[:, 1] submission = pd.DataFrame(preds, index=ids, columns=['target']) with open('BTB_Lasagne.csv', 'a') as f: submission.to_csv(f, header=False) print("Predicting COMPLETE!!")
regression = False, max_epochs = 2, # Number of epochs the training will be done. Higher the value better trained but also risk of overfitting verbose=1, ) # Train the network on the training data - Similar to Scikit-learn fit function net1.fit(train_x_final,train_y_final) # In[8]: # test_check = np.asarray(test, dtype = np.float32) # test_check = test_check.reshape(-1,1,28,28) # test_check predicted = net1.predict_proba(test_check) predicted_nn = pd.DataFrame(predicted, index = test.index, columns = list('0123456789')) predicted_nn.head(10) predicted_nn.to_csv("predicted_nn_iter2_probab.csv") # In[20]: from lasagne.objectives import multiclass_hinge_loss # In[ ]: # scaler = preprocessing.StandardScaler() # train_final_new = scaler.fit_transform(train_final.ix[:,1:])
dropout1_p=0.35, dense2_num_units=180, dropout2_p=0.1, output_num_units=num_classes, output_nonlinearity=softmax, update=adagrad, update_learning_rate=0.0003, # eval_size=0.0, # objective_loss_function = binary_accuracy, verbose=1, max_epochs=1, ) for i in range(epochs): net1.fit(X[:split], y[:split]) pred = net1.predict_proba(X[split:])[:, 1] val_auc[i] = roc_auc_score(y[split:], pred) test = pd.read_csv("test.csv") y2 = test.QuoteNumber.values test = test.drop(["QuoteNumber"], axis=1) # Lets take out some dates test["Date"] = pd.to_datetime(pd.Series(test["Original_Quote_Date"])) test = test.drop("Original_Quote_Date", axis=1) test["Year"] = test["Date"].apply(lambda x: int(str(x)[:4])) test["Month"] = test["Date"].apply(lambda x: int(str(x)[5:7])) test["weekday"] = test["Date"].dt.dayofweek test = train.drop("Date", axis=1)
remainder_test_points = len(X_test) % NO_TIME_POINTS no_rows = total_test_time_points * NO_TIME_POINTS X_test = X_test[0:no_rows, :] X_test = X_test.transpose() X_test_Samples = np.split(X_test, total_test_time_points, axis=1) X_test = np.asarray(X_test_Samples) ########################################################################### #######get predictions and write to files for series 9 and series 10####### print("Testing subject%d...." %(subject)) params = net.get_all_params_values() learned_weights = net.load_params_from(params) probabilities = net.predict_proba(X_test) sub9 = 'subj{0}_series{1}'.format(subject, 9) data_len9 = test_dict[sub9] total_time_points9 = data_len9 // NO_TIME_POINTS remainder_data9 = data_len9 % NO_TIME_POINTS sub10 = 'subj{0}_series{1}'.format(subject, 10) data_len10 = test_dict[sub10] total_time_points10 = data_len10 // NO_TIME_POINTS remainder_data10 = data_len10 % NO_TIME_POINTS total_test_points = total_time_points9+total_time_points10 for i, p in enumerate(probabilities): if i != total_test_points:
dense1_num_units=512, dropout1_p=0.35, dense2_num_units=180, dropout2_p=0.1, output_num_units=num_classes, output_nonlinearity=softmax, update=adagrad, update_learning_rate=0.0003, #eval_size=0.0, # objective_loss_function = binary_accuracy, verbose=1, max_epochs=1) for i in range(epochs): net1.fit(X[:split], y[:split]) pred = net1.predict_proba(X[split:])[:, 1] val_auc[i] = roc_auc_score(y[split:], pred) test = pd.read_csv('test.csv') y2 = test.QuoteNumber.values test = test.drop(['QuoteNumber'], axis=1) # Lets take out some dates test['Date'] = pd.to_datetime(pd.Series(test['Original_Quote_Date'])) test = test.drop('Original_Quote_Date', axis=1) test['Year'] = test['Date'].apply(lambda x: int(str(x)[:4])) test['Month'] = test['Date'].apply(lambda x: int(str(x)[5:7])) test['weekday'] = test['Date'].dt.dayofweek test = train.drop('Date', axis=1)
("input", InputLayer), ("dense0", DenseLayer), ("dropout", DropoutLayer), ("dense1", DenseLayer), ("output", DenseLayer), ] net0 = NeuralNet( layers=layers0, input_shape=(None, num_features), dense0_num_units=100, dropout_p=0.5, dense1_num_units=100, output_num_units=num_classes, output_nonlinearity=softmax, update=nesterov_momentum, update_learning_rate=0.05, update_momentum=0.9, eval_size=0.2, verbose=1, max_epochs=20, ) net0.fit(train_X, train_y) y_prob = net0.predict_proba(check_X) print ("LogLoss {score}".format(score=log_loss(check_y, y_prob)))
for dx in range(-6, 6): for dy in range(-6, 6): if nuclei_map[limit(i + dx, 0, SIZE - 1), limit(j + dy, 0, SIZE - 1)]: cover = True for xx, yy in coords: if i == xx and j == yy: print ("Found coordinate of mitosis: ", (xx, yy)) print ("Cover = ", cover) if cover: patches.append((i, j)) if len(patches) >= BUFFER: print ("Evaluating: ", mitosis_area, nuclei_area) patches2 = get_patches(patches) prob = nn.predict_proba(patches2)[:, 1] nuclei_area += np.sum(prob) mitosis_area += len(prob) patches = [] if len(patches) >= 1: print ("Evaluating!") patches2 = get_patches(patches) prob = nn.predict_proba(patches2)[:, 1] nuclei_area += np.sum(prob) mitosis_area += len(prob) patches = []
count = 0 print("Starting model combination...") for startx in range(4): for starty in range(4): for rotate in range(4): net.batch_iterator_test = AugmentedBatchIterator(batch_size=128, crop_size=4, testing=True, startx=startx, starty=starty, rotate=rotate) y_pred_valid[:, count] = net.predict_proba(X_valid)[:, 1] count += 1 print("Iteration: {} / 64".format(count)) combine = bmc.BMC() combine.fit(y_pred_valid, y_valid) print("Validation set done.") X_test = np.load("../data/sdss_test_images.npy") y_test = np.load("../data/sdss_test_labels.npy") for i in range(5): X_test[:, i, :, :] = renormalize(X_test[:, i, :, :])
) nn.fit(X_train,y_train) X_, X_val, y_, y_val = train_test_split(X_train, y_train, test_size = 25000, random_state = 13) del X_ del y_ xgb = xgboost.XGBClassifier(max_depth = 15, n_estimators = 200, objective='multi:softprob', subsample = .80, colsample_bytree=.5) xgb.fit(X_train, y_train, eval_set = [(X_val, y_val)], eval_metric = 'mlogloss', early_stopping_rounds=25) y_xgb_train_predictions = xgb.predict_proba(X_train) y_nn_train_predictions = nn.predict_proba(X_train) X_ensembl_train = np.concatenate((y_xgb_train_predictions, y_nn_train_predictions), axis = 1) y_nn_test_predictions = nn.predict_proba(X_test) y_xgb_test_predictions = xgb.predict_proba(X_test) col_names = ['TripType_' + str(c) for c in enc.classes_.astype('int')] submission = pd.DataFrame(np.round(y_nn_test_predictions, 4), index=y_df[pd.isnull(y_df.TripType)].index, columns = col_names) submission.reset_index(inplace = True) submission.to_csv('Walmart_ensembl_NN_10000Features-Notebook.csv', index=False) submission = pd.DataFrame(np.round(y_xgb_test_predictions, 4), index=y_df[pd.isnull(y_df.TripType)].index, columns = col_names) submission.reset_index(inplace = True)
class TwoLayerNeuralNetwork(BaseEstimator): def __init__(self, **params): self.seed = params.get('seed', 777) # Set random seed random.seed(self.seed) np.random.seed(self.seed) # Prepare network parameters network_params = {} # Input shape network_params['input_shape'] = (None, params['num_features']) # Prepare layers network_params['layers'] = [] for param_layer in params['layers']: network_params['layers'].append(( param_layer['name'], dynamic_load(param_layer['class']) )) for k in param_layer.keys(): if k in ['name', 'class']: continue name = "{0}_{1}".format(param_layer['name'], k) network_params[name] = param_layer[k] # Prepare outputs network_params['output_nonlinearity'] = ( dynamic_load(params['output_nonlinearity'])) # Prepare updates if 'update' in params: network_params['update'] = dynamic_load(params['update']) if 'update_learning_rate' in params: update_learning_rate = params['update_learning_rate'] network_params['update_learning_rate'] = ( theano.shared(np.cast['float32'](update_learning_rate)) ) if 'adjust_variable' in params: if params['adjust_variable'] == 1: adjuster_start = params.get('adjuster_start', 0.01) adjuster_stop = params.get('adjuster_stop', 0.001) network_params['on_epoch_finished'] = [ AdjustVariable('update_learning_rate', start=adjuster_start, stop=adjuster_stop) ] # Other parameters network_params['eval_size'] = params.get('eval_size', 0.01) network_params['verbose'] = params.get('verbose', 1) network_params['max_epochs'] = params.get('max_epochs', 100) # Create a clf object self.clf = NeuralNet(**network_params) def fit(self, X_train, y_train): self.clf.fit(X_train, y_train) def predict_proba(self, X_test): return self.clf.predict_proba(X_test)