def __init__(self, num_features=93, num_classes=9): """ """ self.prefix = 'nl_simp' self.num_features = num_features self.num_classes = num_classes self.layers = [('input', InputLayer), ('dropouti', DropoutLayer), ('dense0', DenseLayer), ('dropout0', DropoutLayer), ('dense1', DenseLayer), ('dropout1', DropoutLayer), ('dense2', DenseLayer), ('dropout2', DropoutLayer), ('dense3', DenseLayer), ('dropout3', DropoutLayer), ('output', DenseLayer)] self.early_stopping = EarlyStopping(patience=25) self.nn = My_NeuralNet(layers=self.layers, input_shape=(None, self.num_features), dropouti_p=0.12, dense0_num_units=900, dense0_W=HeNormal(), dense0_nonlinearity = LeakyRectify(leakiness=0.002), dropout0_p=0.35, dense1_num_units=600, dense1_W=HeNormal(), dense1_nonlinearity = LeakyRectify(leakiness=0.002), dropout1_p=0.2, dense2_num_units=400, dense2_W=HeNormal(), dense2_nonlinearity = LeakyRectify(leakiness=0.002), dropout2_p=0.1, dense3_num_units=300, dense3_W=HeNormal(), dense3_nonlinearity = LeakyRectify(leakiness=0.002), dropout3_p=0.1, output_num_units=num_classes, output_nonlinearity=softmax, update=adagrad, update_learning_rate=shared(float32(0.01)), batch_iterator_train=BatchIterator(batch_size=512), on_epoch_finished=[ AdjustVariable('update_learning_rate', start=0.01, stop=0.005), # AdjustVariable('update_momentum', start=0.1, stop=0.0), self.early_stopping, ], eval_size=0., verbose=1, max_epochs=300 ) self.nn2 = deepcopy(self.nn)
class Clf_nolearn_simple(Clf_nolearn): """ Simple nolearn based classifier... """ def __init__(self, num_features=93, num_classes=9): """ """ self.prefix = 'nl_simp' self.num_features = num_features self.num_classes = num_classes self.layers = [('input', InputLayer), ('dropouti', DropoutLayer), ('dense0', DenseLayer), ('dropout0', DropoutLayer), ('dense1', DenseLayer), ('dropout1', DropoutLayer), ('dense2', DenseLayer), ('dropout2', DropoutLayer), ('dense3', DenseLayer), ('dropout3', DropoutLayer), ('output', DenseLayer)] self.early_stopping = EarlyStopping(patience=25) self.nn = My_NeuralNet(layers=self.layers, input_shape=(None, self.num_features), dropouti_p=0.12, dense0_num_units=900, dense0_W=HeNormal(), dense0_nonlinearity = LeakyRectify(leakiness=0.002), dropout0_p=0.35, dense1_num_units=600, dense1_W=HeNormal(), dense1_nonlinearity = LeakyRectify(leakiness=0.002), dropout1_p=0.2, dense2_num_units=400, dense2_W=HeNormal(), dense2_nonlinearity = LeakyRectify(leakiness=0.002), dropout2_p=0.1, dense3_num_units=300, dense3_W=HeNormal(), dense3_nonlinearity = LeakyRectify(leakiness=0.002), dropout3_p=0.1, output_num_units=num_classes, output_nonlinearity=softmax, update=adagrad, update_learning_rate=shared(float32(0.01)), batch_iterator_train=BatchIterator(batch_size=512), on_epoch_finished=[ AdjustVariable('update_learning_rate', start=0.01, stop=0.005), # AdjustVariable('update_momentum', start=0.1, stop=0.0), self.early_stopping, ], eval_size=0., verbose=1, max_epochs=300 ) self.nn2 = deepcopy(self.nn) def train_validate(self, X_train, y_train, X_valid, y_valid): """ """ self.nn.max_epochs = 300 self.nn.verbose=1 self.nn.fit(X_train, y_train, X_valid, y_valid) # XX = np.vstack((X_train, X_valid[:len(X_valid)/2])) # yy = np.hstack((y_train, y_valid[:len(y_valid)/2])) # XXv = X_valid[len(X_valid)/2:] # yyv = y_valid[len(y_valid)/2:] ## self.nn.dropouti_p=0.25 # self.nn.fit(XX, yy, XXv, yyv) self.nn2.fit(X_train, y_train, X_valid, y_valid) # self.nn.fit(X_train, y_train) yp0 = self.nn.predict_proba(X_valid) print 'Nolearn log-loss: %s'%(logloss_mc(y_valid, yp0)) # self.nn.max_epochs = self.early_stopping.best_valid_epoch # print self.early_stopping.best_valid_epoch # self.nn.verbose=0 # # clf = ClfCal(self.nn) # cc = CalibratedClassifierCV(base_estimator=clf, method='isotonic', # cv=StratifiedKFold(y_train, n_folds=3)) # cc.fit(X_train, y_train) # yp1= cc.predict_proba(X_valid) # print 'Calibrated log-loss: %s' %(logloss_mc(y_valid, yp1)) # y_pred = (yp0+yp1)/2. # print 'Mean log-loss: %s' %(logloss_mc(y_valid, y_pred)) # # self.cal_clf = cc y_pred = yp0 # pdb.set_trace() return y_pred def train_test(self, X, y, X_test): """ """ self.nn.max_epochs = self.early_stopping.best_valid_epoch # self.nn.verbose=1 # self.nn.eval_size=0. # self.nn.on_epoch_finished=[ # AdjustVariable('update_learning_rate', start=0.1, # stop=0.001) # ] self.nn.fit(X, y) yp0 = self.nn.predict_proba(X_test) # self.cal_clf.fit(X, y) # yp1 = self.cal_clf.predict_proba(X_test) # y_pred = (yp0 + yp1)/2. y_pred = yp0 return y_pred
def __init__(self, num_features=93, num_classes=9): """ """ self.prefix = 'nl_2lv_cal' self.num_features = num_features self.num_classes = num_classes self.layers0 = [('input', InputLayer), ('dropoutn', DropoutLayer), ('dense0', DenseLayer), ('dropout0', DropoutLayer), ('dense1', DenseLayer), ('dropout1', DropoutLayer), ('dense2', DenseLayer), ('dropout2', DropoutLayer), ('dense3', DenseLayer), ('dropout3', DropoutLayer), ('output', DenseLayer)] self.early_stopping0 = EarlyStopping(patience=20) self.nn0 = My_NeuralNet(layers=self.layers0, input_shape=(None, self.num_features), dropoutn_p=0.12, dense0_num_units=900, dense0_W=HeNormal(), dense0_nonlinearity = LeakyRectify(leakiness=0.002), dropout0_p=0.35, dense1_num_units=600, dense1_W=HeNormal(), dense1_nonlinearity = LeakyRectify(leakiness=0.002), dropout1_p=0.2, dense2_num_units=400, dense2_W=HeNormal(), dense2_nonlinearity = LeakyRectify(leakiness=0.002), dropout2_p=0.1, dense3_num_units=300, dense3_W=HeNormal(), dense3_nonlinearity = LeakyRectify(leakiness=0.002), dropout3_p=0.1, output_num_units=2, output_nonlinearity=softmax, update=adagrad, update_learning_rate=shared(float32(0.01)), batch_iterator_train=BatchIterator(batch_size=256), on_epoch_finished=[ AdjustVariable('update_learning_rate', start=0.01, stop=0.001), # AdjustVariable('update_momentum', start=0.1, stop=0.0), self.early_stopping0, ], eval_size=0, verbose=1, max_epochs=36 ) self.layers1 = [('input', InputLayer), ('dropoutn', DropoutLayer), ('dense0', DenseLayer), # ('gaussian0', GaussianNoiseLayer), ('dropout0', DropoutLayer), ('dense1', DenseLayer), # ('gaussian1', GaussianNoiseLayer), ('dropout1', DropoutLayer), ('dense2', DenseLayer), ('dropout2', DropoutLayer), # ('dense3', DenseLayer), # ('dropout3', DropoutLayer), ('output', DenseLayer)] self.early_stopping1 = EarlyStopping(patience=20) self.nn1 = My_NeuralNet(layers=self.layers1, input_shape=(None, self.num_features), dropoutn_p=0.18, dense0_num_units=900, dense0_W=HeNormal(), dense0_nonlinearity = LeakyRectify(leakiness=0.002), dropout0_p=0.35, dense1_num_units=600, dense1_W=HeNormal(), dense1_nonlinearity = LeakyRectify(leakiness=0.002), dropout1_p=0.15, dense2_num_units=400, dense2_W=HeNormal(), dense2_nonlinearity = LeakyRectify(leakiness=0.002), dropout2_p=0.05, output_num_units=3, output_nonlinearity=softmax, update=adagrad, update_learning_rate=shared(float32(0.01)), batch_iterator_train=BatchIterator(batch_size=256), on_epoch_finished=[ AdjustVariable('update_learning_rate', start=0.01, stop=0.001), # AdjustVariable('update_momentum', start=0.1, stop=0.0), self.early_stopping1, ], eval_size=0, verbose=1, max_epochs=60 ) self.layers2 = [('input', InputLayer), ('dropoutn', DropoutLayer), ('dense0', DenseLayer), # ('gaussian0', GaussianNoiseLayer), ('dropout0', DropoutLayer), ('dense1', DenseLayer), # ('gaussian1', GaussianNoiseLayer), ('dropout1', DropoutLayer), ('dense2', DenseLayer), ('dropout2', DropoutLayer), # ('dense3', DenseLayer), # ('dropout3', DropoutLayer), ('output', DenseLayer)] self.early_stopping2 = EarlyStopping(patience=20) self.nn2 = My_NeuralNet(layers=self.layers2, input_shape=(None, self.num_features), dropoutn_p=0.13, dense0_num_units=900, dense0_W=HeNormal(), dense0_nonlinearity = LeakyRectify(leakiness=0.002), dropout0_p=0.35, dense1_num_units=600, dense1_W=HeNormal(), dense1_nonlinearity = LeakyRectify(leakiness=0.002), dropout1_p=0.15, dense2_num_units=400, dense2_W=HeNormal(), dense2_nonlinearity = LeakyRectify(leakiness=0.002), dropout2_p=0.05, output_num_units=6, output_nonlinearity=softmax, update=adagrad, update_learning_rate=shared(float32(0.01)), batch_iterator_train=BatchIterator(batch_size=256), on_epoch_finished=[ AdjustVariable('update_learning_rate', start=0.01, stop=0.001), # AdjustVariable('update_momentum', start=0.1, stop=0.0), self.early_stopping2, ], # eval_size=0.2, eval_size=0, verbose=1, max_epochs=31 )
class Clf_nolearn_2_levels_cal(Clf_nolearn): """ Simple nolearn based classifier... """ def __init__(self, num_features=93, num_classes=9): """ """ self.prefix = 'nl_2lv_cal' self.num_features = num_features self.num_classes = num_classes self.layers0 = [('input', InputLayer), ('dropoutn', DropoutLayer), ('dense0', DenseLayer), ('dropout0', DropoutLayer), ('dense1', DenseLayer), ('dropout1', DropoutLayer), ('dense2', DenseLayer), ('dropout2', DropoutLayer), ('dense3', DenseLayer), ('dropout3', DropoutLayer), ('output', DenseLayer)] self.early_stopping0 = EarlyStopping(patience=20) self.nn0 = My_NeuralNet(layers=self.layers0, input_shape=(None, self.num_features), dropoutn_p=0.12, dense0_num_units=900, dense0_W=HeNormal(), dense0_nonlinearity = LeakyRectify(leakiness=0.002), dropout0_p=0.35, dense1_num_units=600, dense1_W=HeNormal(), dense1_nonlinearity = LeakyRectify(leakiness=0.002), dropout1_p=0.2, dense2_num_units=400, dense2_W=HeNormal(), dense2_nonlinearity = LeakyRectify(leakiness=0.002), dropout2_p=0.1, dense3_num_units=300, dense3_W=HeNormal(), dense3_nonlinearity = LeakyRectify(leakiness=0.002), dropout3_p=0.1, output_num_units=2, output_nonlinearity=softmax, update=adagrad, update_learning_rate=shared(float32(0.01)), batch_iterator_train=BatchIterator(batch_size=256), on_epoch_finished=[ AdjustVariable('update_learning_rate', start=0.01, stop=0.001), # AdjustVariable('update_momentum', start=0.1, stop=0.0), self.early_stopping0, ], eval_size=0, verbose=1, max_epochs=36 ) self.layers1 = [('input', InputLayer), ('dropoutn', DropoutLayer), ('dense0', DenseLayer), # ('gaussian0', GaussianNoiseLayer), ('dropout0', DropoutLayer), ('dense1', DenseLayer), # ('gaussian1', GaussianNoiseLayer), ('dropout1', DropoutLayer), ('dense2', DenseLayer), ('dropout2', DropoutLayer), # ('dense3', DenseLayer), # ('dropout3', DropoutLayer), ('output', DenseLayer)] self.early_stopping1 = EarlyStopping(patience=20) self.nn1 = My_NeuralNet(layers=self.layers1, input_shape=(None, self.num_features), dropoutn_p=0.18, dense0_num_units=900, dense0_W=HeNormal(), dense0_nonlinearity = LeakyRectify(leakiness=0.002), dropout0_p=0.35, dense1_num_units=600, dense1_W=HeNormal(), dense1_nonlinearity = LeakyRectify(leakiness=0.002), dropout1_p=0.15, dense2_num_units=400, dense2_W=HeNormal(), dense2_nonlinearity = LeakyRectify(leakiness=0.002), dropout2_p=0.05, output_num_units=3, output_nonlinearity=softmax, update=adagrad, update_learning_rate=shared(float32(0.01)), batch_iterator_train=BatchIterator(batch_size=256), on_epoch_finished=[ AdjustVariable('update_learning_rate', start=0.01, stop=0.001), # AdjustVariable('update_momentum', start=0.1, stop=0.0), self.early_stopping1, ], eval_size=0, verbose=1, max_epochs=60 ) self.layers2 = [('input', InputLayer), ('dropoutn', DropoutLayer), ('dense0', DenseLayer), # ('gaussian0', GaussianNoiseLayer), ('dropout0', DropoutLayer), ('dense1', DenseLayer), # ('gaussian1', GaussianNoiseLayer), ('dropout1', DropoutLayer), ('dense2', DenseLayer), ('dropout2', DropoutLayer), # ('dense3', DenseLayer), # ('dropout3', DropoutLayer), ('output', DenseLayer)] self.early_stopping2 = EarlyStopping(patience=20) self.nn2 = My_NeuralNet(layers=self.layers2, input_shape=(None, self.num_features), dropoutn_p=0.13, dense0_num_units=900, dense0_W=HeNormal(), dense0_nonlinearity = LeakyRectify(leakiness=0.002), dropout0_p=0.35, dense1_num_units=600, dense1_W=HeNormal(), dense1_nonlinearity = LeakyRectify(leakiness=0.002), dropout1_p=0.15, dense2_num_units=400, dense2_W=HeNormal(), dense2_nonlinearity = LeakyRectify(leakiness=0.002), dropout2_p=0.05, output_num_units=6, output_nonlinearity=softmax, update=adagrad, update_learning_rate=shared(float32(0.01)), batch_iterator_train=BatchIterator(batch_size=256), on_epoch_finished=[ AdjustVariable('update_learning_rate', start=0.01, stop=0.001), # AdjustVariable('update_momentum', start=0.1, stop=0.0), self.early_stopping2, ], # eval_size=0.2, eval_size=0, verbose=1, max_epochs=31 ) def train_validate(self, X_train, y_train, X_valid, y_valid): """ """ le = LabelEncoder() id_123 = np.logical_or(np.logical_or(y_train==1, y_train==2), y_train==3) y0 = np.zeros(len(y_train), dtype=np.int32) y0[id_123] = 1 X0 = np.copy(X_train) y0 = le.fit_transform(y0).astype(np.int32) X1 = X_train[id_123] y1 = y_train[id_123] y1 = le.fit_transform(y1).astype(np.int32) X2 = X_train[np.logical_not(id_123)] y2 = y_train[np.logical_not(id_123)] y2 = le.fit_transform(y2).astype(np.int32) #Validation id_123_valid = np.logical_or(np.logical_or(y_valid==1, y_valid==2), y_valid==3) y0_valid = np.zeros(len(y_valid), dtype=np.int32) y0_valid[id_123_valid] = 1 X0_valid = np.copy(X_valid) y0_valid = le.fit_transform(y0_valid).astype(np.int32) X1_valid = X_valid[id_123_valid] y1_valid = y_valid[id_123_valid] y1_valid = le.fit_transform(y1_valid).astype(np.int32) X2_valid = X_valid[np.logical_not(id_123_valid)] y2_valid = y_valid[np.logical_not(id_123_valid)] y2_valid = le.fit_transform(y2_valid).astype(np.int32) self.nn0.max_epochs = 300 self.nn0.verbose=1 self.nn0.fit(X0, y0, X0_valid, y0_valid) y0_pred = self.nn0.predict_proba(X_valid) self.nn1.max_epochs = 300 self.nn1.verbose=1 self.nn1.fit(X1, y1, X1_valid, y1_valid) y1_pred = self.nn1.predict_proba(X_valid) self.nn2.max_epochs = 300 self.nn2.verbose=1 self.nn2.fit(X2, y2, X2_valid, y2_valid) y2_pred = self.nn2.predict_proba(X_valid) y_pred = np.zeros((y0_pred.shape[0], 9)) y_pred[:,0] = y0_pred[:,0]*y2_pred[:,0] y_pred[:,1] = y0_pred[:,1]*y1_pred[:,0] y_pred[:,2] = y0_pred[:,1]*y1_pred[:,1] y_pred[:,3] = y0_pred[:,1]*y1_pred[:,2] y_pred[:,4] = y0_pred[:,0]*y2_pred[:,1] y_pred[:,5] = y0_pred[:,0]*y2_pred[:,2] y_pred[:,6] = y0_pred[:,0]*y2_pred[:,3] y_pred[:,7] = y0_pred[:,0]*y2_pred[:,4] y_pred[:,8] = y0_pred[:,0]*y2_pred[:,5] yp0 = y_pred print logloss_mc(y_valid, yp0) self.nn0.max_epochs = self.early_stopping0.best_valid_epoch self.nn0.verbose=0 self.nn1.max_epochs = self.early_stopping1.best_valid_epoch self.nn1.verbose=0 self.nn2.max_epochs = self.early_stopping2.best_valid_epoch self.nn2.verbose=0 clf = ClfCal(self.nn0, self.nn1, self.nn2) cc = CalibratedClassifierCV(base_estimator=clf, method='isotonic', cv=StratifiedKFold(y_train, n_folds=3)) cc.fit(X_train, y_train) yp1= cc.predict_proba(X_valid) print 'Calibrated log-loss: %s' %(logloss_mc(y_valid, yp1)) y_pred = (yp0+yp1)/2. print 'Mean log-loss: %s' %(logloss_mc(y_valid, y_pred)) self.cal_clf = cc return y_pred def train_test(self, X, y, X_test): """ """ le = LabelEncoder() id_123 = np.logical_or(np.logical_or(y==1, y==2), y==3) y0 = np.zeros(len(y), dtype=np.int32) y0[id_123] = 1 X0 = np.copy(X) y0 = le.fit_transform(y0).astype(np.int32) X1 = X[id_123] y1 = y[id_123] y1 = le.fit_transform(y1).astype(np.int32) X2 = X[np.logical_not(id_123)] y2 = y[np.logical_not(id_123)] y2 = le.fit_transform(y2).astype(np.int32) print 'working on nn0...' self.nn0.max_epochs = self.early_stopping0.best_valid_epoch self.nn0.verbose=0 self.nn0.fit(X0, y0) y0_pred = self.nn0.predict_proba(X_test) print 'working on nn1...' self.nn1.max_epochs = self.early_stopping1.best_valid_epoch self.nn1.verbose=0 self.nn1.fit(X1, y1) y1_pred = self.nn1.predict_proba(X_test) print 'working on nn2...' self.nn2.max_epochs = self.early_stopping2.best_valid_epoch self.nn2.verbose=0 self.nn2.fit(X2, y2) y2_pred = self.nn2.predict_proba(X_test) y_pred = np.zeros((y0_pred.shape[0], 9)) y_pred[:,0] = y0_pred[:,0]*y2_pred[:,0] y_pred[:,1] = y0_pred[:,1]*y1_pred[:,0] y_pred[:,2] = y0_pred[:,1]*y1_pred[:,1] y_pred[:,3] = y0_pred[:,1]*y1_pred[:,2] y_pred[:,4] = y0_pred[:,0]*y2_pred[:,1] y_pred[:,5] = y0_pred[:,0]*y2_pred[:,2] y_pred[:,6] = y0_pred[:,0]*y2_pred[:,3] y_pred[:,7] = y0_pred[:,0]*y2_pred[:,4] y_pred[:,8] = y0_pred[:,0]*y2_pred[:,5] yp0 = y_pred self.cal_clf.fit(X, y) yp1 = self.cal_clf.predict_proba(X_test) y_pred = (yp0 + yp1)/2. return y_pred
class Clf_nolearn_simple_play(Clf_nolearn): """ Simple nolearn based classifier... """ def __init__(self, num_features=93, num_classes=9): """ """ self.prefix = 'nl_simp_play' self.num_features = num_features self.num_classes = num_classes self.layers = [('input', InputLayer), ('dropouti', DropoutLayer), ('dense0', DenseLayer), ('dropout0', DropoutLayer), ('dense1', DenseLayer), ('dropout1', DropoutLayer), ('dense2', DenseLayer), ('dropout2', DropoutLayer), ('dense3', DenseLayer), ('dropout3', DropoutLayer), ('output', DenseLayer)] # self.early_stopping = EarlyStopping(patience=25) self.oneone_stopping = OneOneStopping(ratio=0.9) self.oneone_stopping2 = OneOneStopping(ratio=0.9) self.oneone_stopping3 = OneOneStopping(ratio=0.85) self.nn = My_NeuralNet(layers=self.layers, input_shape=(None, self.num_features), dropouti_p=0.1, dense0_num_units=900, dense0_W=HeNormal(), dense0_nonlinearity = LeakyRectify(leakiness=0.05), dropout0_p=0.45, dense1_num_units=600, dense1_W=HeNormal(), dense1_nonlinearity = LeakyRectify(leakiness=0.05), dropout1_p=0.3, dense2_num_units=400, dense2_W=HeNormal(), dense2_nonlinearity = LeakyRectify(leakiness=0.05), dropout2_p=0.2, dense3_num_units=300, dense3_W=HeNormal(), dense3_nonlinearity = LeakyRectify(leakiness=0.05), dropout3_p=0.1, output_num_units=num_classes, output_nonlinearity=softmax, update=adagrad, update_learning_rate=shared(float32(0.01)), batch_iterator_train=BatchIterator(batch_size=256), on_epoch_finished=[ AdjustVariable('update_learning_rate', start=0.01, stop=0.005), # AdjustVariable('update_momentum', start=0.1, stop=0.0), self.oneone_stopping, ], eval_size=0., verbose=1, max_epochs=400 ) self.nnt = deepcopy(self.nn) self.nn2 = My_NeuralNet(layers=self.layers, input_shape=(None, self.num_features), dropouti_p=0.12, dense0_num_units=900, dense0_W=HeNormal(), dense0_nonlinearity = LeakyRectify(leakiness=0.05), dropout0_p=0.47, dense1_num_units=600, dense1_W=HeNormal(), dense1_nonlinearity = LeakyRectify(leakiness=0.05), dropout1_p=0.32, dense2_num_units=400, dense2_W=HeNormal(), dense2_nonlinearity = LeakyRectify(leakiness=0.05), dropout2_p=0.22, dense3_num_units=300, dense3_W=HeNormal(), dense3_nonlinearity = LeakyRectify(leakiness=0.05), dropout3_p=0.12, output_num_units=self.num_classes, output_nonlinearity=softmax, update=adagrad, update_learning_rate=shared(float32(0.01)), batch_iterator_train=BatchIterator(batch_size=256), on_epoch_finished=[ AdjustVariable('update_learning_rate', start=0.01, stop=0.005), # AdjustVariable('update_momentum', start=0.1, stop=0.0), # self.early_stopping2, self.oneone_stopping2, ], eval_size=0., verbose=1, max_epochs=400 ) self.nn2t = deepcopy(self.nn2) self.nn3 = My_NeuralNet(layers=self.layers, input_shape=(None, self.num_features), dropouti_p=0.14, dense0_num_units=900, dense0_W=HeNormal(), dense0_nonlinearity = LeakyRectify(leakiness=0.05), dropout0_p=0.49, dense1_num_units=600, dense1_W=HeNormal(), dense1_nonlinearity = LeakyRectify(leakiness=0.05), dropout1_p=0.34, dense2_num_units=400, dense2_W=HeNormal(), dense2_nonlinearity = LeakyRectify(leakiness=0.05), dropout2_p=0.24, dense3_num_units=300, dense3_W=HeNormal(), dense3_nonlinearity = LeakyRectify(leakiness=0.05), dropout3_p=0.14, output_num_units=self.num_classes, output_nonlinearity=softmax, update=adagrad, update_learning_rate=shared(float32(0.01)), batch_iterator_train=BatchIterator(batch_size=256), on_epoch_finished=[ AdjustVariable('update_learning_rate', start=0.01, stop=0.005), # AdjustVariable('update_momentum', start=0.1, stop=0.0), # self.early_stopping, self.oneone_stopping3, ], eval_size=0., verbose=1, max_epochs=400 ) self.nn3t = deepcopy(self.nn3) def train_validate(self, X_train, y_train, X_valid, y_valid): """ """ self.nn.max_epochs = 300 self.nn.verbose=1 self.nn.fit(X_train, y_train, X_valid, y_valid) params = self.nn.get_all_params_values() self.nn2.load_params_from(params) self.nn2.fit(X_train, y_train, X_valid, y_valid) params2 = self.nn2.get_all_params_values() self.nn3.load_params_from(params2) self.nn3.fit(X_train, y_train, X_valid, y_valid) yp0 = self.nn3.predict_proba(X_valid) print 'Nolearn log-loss: %s'%(logloss_mc(y_valid, yp0)) y_pred = yp0 # pdb.set_trace() return y_pred def train_test(self, X, y, X_test): """ """ self.nnt.max_epochs = self.oneone_stopping.best_valid_epoch self.nnt.fit(X, y) params = self.nnt.get_all_params_values() self.nn2t.load_params_from(params) self.nn2t.max_epochs = self.oneone_stopping2.best_valid_epoch self.nn2t.fit(X, y) params2 = self.nn2t.get_all_params_values() self.nn3t.load_params_from(params2) self.nn3t.max_epochs = self.oneone_stopping3.best_valid_epoch self.nn3t.fit(X, y) yp0 = self.nn3t.predict_proba(X_test) # self.cal_clf.fit(X, y) # yp1 = self.cal_clf.predict_proba(X_test) # y_pred = (yp0 + yp1)/2. y_pred = yp0 return y_pred
class Clf_nolearn_2_levels(Clf_nolearn): """ Simple nolearn based classifier... """ def __init__(self, num_features=93, num_classes=9): """ """ self.prefix = "nl_2lv" self.num_features = num_features self.num_classes = num_classes self.layers0 = [ ("input", InputLayer), ("dropoutn", DropoutLayer), ("dense0", DenseLayer), ("dropout0", DropoutLayer), ("dense1", DenseLayer), ("dropout1", DropoutLayer), ("dense2", DenseLayer), ("dropout2", DropoutLayer), ("dense3", DenseLayer), ("dropout3", DropoutLayer), ("output", DenseLayer), ] self.early_stopping0 = EarlyStopping(patience=20) self.nn0 = My_NeuralNet( layers=self.layers0, input_shape=(None, self.num_features), dropoutn_p=0.12, dense0_num_units=900, dense0_W=HeNormal(), dense0_nonlinearity=LeakyRectify(leakiness=0.002), dropout0_p=0.35, dense1_num_units=600, dense1_W=HeNormal(), dense1_nonlinearity=LeakyRectify(leakiness=0.002), dropout1_p=0.2, dense2_num_units=400, dense2_W=HeNormal(), dense2_nonlinearity=LeakyRectify(leakiness=0.002), dropout2_p=0.1, dense3_num_units=300, dense3_W=HeNormal(), dense3_nonlinearity=LeakyRectify(leakiness=0.002), dropout3_p=0.1, output_num_units=2, output_nonlinearity=softmax, update=adagrad, update_learning_rate=shared(float32(0.01)), batch_iterator_train=BatchIterator(batch_size=64), on_epoch_finished=[ AdjustVariable("update_learning_rate", start=0.01, stop=0.001), # AdjustVariable('update_momentum', start=0.1, stop=0.0), self.early_stopping0, ], eval_size=0, verbose=1, max_epochs=36, ) self.layers1 = [ ("input", InputLayer), ("dropoutn", DropoutLayer), ("dense0", DenseLayer), # ('gaussian0', GaussianNoiseLayer), ("dropout0", DropoutLayer), ("dense1", DenseLayer), # ('gaussian1', GaussianNoiseLayer), ("dropout1", DropoutLayer), ("dense2", DenseLayer), ("dropout2", DropoutLayer), # ('dense3', DenseLayer), # ('dropout3', DropoutLayer), ("output", DenseLayer), ] self.early_stopping1 = EarlyStopping(patience=20) self.nn1 = My_NeuralNet( layers=self.layers1, input_shape=(None, self.num_features), dropoutn_p=0.18, dense0_num_units=900, dense0_W=HeNormal(), dense0_nonlinearity=LeakyRectify(leakiness=0.002), dropout0_p=0.35, dense1_num_units=600, dense1_W=HeNormal(), dense1_nonlinearity=LeakyRectify(leakiness=0.002), dropout1_p=0.15, dense2_num_units=400, dense2_W=HeNormal(), dense2_nonlinearity=LeakyRectify(leakiness=0.002), dropout2_p=0.05, output_num_units=3, output_nonlinearity=softmax, update=adagrad, update_learning_rate=shared(float32(0.01)), batch_iterator_train=BatchIterator(batch_size=64), on_epoch_finished=[ AdjustVariable("update_learning_rate", start=0.01, stop=0.001), # AdjustVariable('update_momentum', start=0.1, stop=0.0), self.early_stopping1, ], eval_size=0, verbose=1, max_epochs=60, ) self.layers2 = [ ("input", InputLayer), ("dropoutn", DropoutLayer), ("dense0", DenseLayer), # ('gaussian0', GaussianNoiseLayer), ("dropout0", DropoutLayer), ("dense1", DenseLayer), # ('gaussian1', GaussianNoiseLayer), ("dropout1", DropoutLayer), ("dense2", DenseLayer), ("dropout2", DropoutLayer), # ('dense3', DenseLayer), # ('dropout3', DropoutLayer), ("output", DenseLayer), ] self.early_stopping2 = EarlyStopping(patience=20) self.nn2 = My_NeuralNet( layers=self.layers2, input_shape=(None, self.num_features), dropoutn_p=0.13, dense0_num_units=900, dense0_W=HeNormal(), dense0_nonlinearity=LeakyRectify(leakiness=0.002), dropout0_p=0.35, dense1_num_units=600, dense1_W=HeNormal(), dense1_nonlinearity=LeakyRectify(leakiness=0.002), dropout1_p=0.15, dense2_num_units=400, dense2_W=HeNormal(), dense2_nonlinearity=LeakyRectify(leakiness=0.002), dropout2_p=0.05, output_num_units=6, output_nonlinearity=softmax, update=adagrad, update_learning_rate=shared(float32(0.01)), batch_iterator_train=BatchIterator(batch_size=64), on_epoch_finished=[ AdjustVariable("update_learning_rate", start=0.01, stop=0.001), # AdjustVariable('update_momentum', start=0.1, stop=0.0), self.early_stopping2, ], # eval_size=0.2, eval_size=0, verbose=1, max_epochs=31, ) def train_validate(self, X_train, y_train, X_valid, y_valid): """ """ le = LabelEncoder() id_123 = np.logical_or(np.logical_or(y_train == 1, y_train == 2), y_train == 3) y0 = np.zeros(len(y_train), dtype=np.int32) y0[id_123] = 1 X0 = np.copy(X_train) y0 = le.fit_transform(y0).astype(np.int32) X1 = X_train[id_123] y1 = y_train[id_123] y1 = le.fit_transform(y1).astype(np.int32) X2 = X_train[np.logical_not(id_123)] y2 = y_train[np.logical_not(id_123)] y2 = le.fit_transform(y2).astype(np.int32) # Validation id_123_valid = np.logical_or(np.logical_or(y_valid == 1, y_valid == 2), y_valid == 3) y0_valid = np.zeros(len(y_valid), dtype=np.int32) y0_valid[id_123_valid] = 1 X0_valid = np.copy(X_valid) y0_valid = le.fit_transform(y0_valid).astype(np.int32) X1_valid = X_valid[id_123_valid] y1_valid = y_valid[id_123_valid] y1_valid = le.fit_transform(y1_valid).astype(np.int32) X2_valid = X_valid[np.logical_not(id_123_valid)] y2_valid = y_valid[np.logical_not(id_123_valid)] y2_valid = le.fit_transform(y2_valid).astype(np.int32) self.nn0.max_epochs = 300 self.nn0.verbose = 1 self.nn0.fit(X0, y0, X0_valid, y0_valid) y0_pred = self.nn0.predict_proba(X_valid) self.nn1.max_epochs = 300 self.nn1.verbose = 1 self.nn1.fit(X1, y1, X1_valid, y1_valid) y1_pred = self.nn1.predict_proba(X_valid) self.nn2.max_epochs = 300 self.nn2.verbose = 1 self.nn2.fit(X2, y2, X2_valid, y2_valid) y2_pred = self.nn2.predict_proba(X_valid) y_pred = np.zeros((y0_pred.shape[0], 9)) y_pred[:, 0] = y0_pred[:, 0] * y2_pred[:, 0] y_pred[:, 1] = y0_pred[:, 1] * y1_pred[:, 0] y_pred[:, 2] = y0_pred[:, 1] * y1_pred[:, 1] y_pred[:, 3] = y0_pred[:, 1] * y1_pred[:, 2] y_pred[:, 4] = y0_pred[:, 0] * y2_pred[:, 1] y_pred[:, 5] = y0_pred[:, 0] * y2_pred[:, 2] y_pred[:, 6] = y0_pred[:, 0] * y2_pred[:, 3] y_pred[:, 7] = y0_pred[:, 0] * y2_pred[:, 4] y_pred[:, 8] = y0_pred[:, 0] * y2_pred[:, 5] return y_pred def train_test(self, X, y, X_test): """ """ le = LabelEncoder() id_123 = np.logical_or(np.logical_or(y == 1, y == 2), y == 3) y0 = np.zeros(len(y), dtype=np.int32) y0[id_123] = 1 X0 = np.copy(X) y0 = le.fit_transform(y0).astype(np.int32) X1 = X[id_123] y1 = y[id_123] y1 = le.fit_transform(y1).astype(np.int32) X2 = X[np.logical_not(id_123)] y2 = y[np.logical_not(id_123)] y2 = le.fit_transform(y2).astype(np.int32) print "working on nn0..." self.nn0.max_epochs = self.early_stopping0.best_valid_epoch self.nn0.verbose = 0 self.nn0.fit(X0, y0) y0_pred = self.nn0.predict_proba(X_test) print "working on nn1..." self.nn1.max_epochs = self.early_stopping1.best_valid_epoch self.nn1.verbose = 0 self.nn1.fit(X1, y1) y1_pred = self.nn1.predict_proba(X_test) print "working on nn2..." self.nn2.max_epochs = self.early_stopping2.best_valid_epoch self.nn2.verbose = 0 self.nn2.fit(X2, y2) y2_pred = self.nn2.predict_proba(X_test) y_pred = np.zeros((y0_pred.shape[0], 9)) y_pred[:, 0] = y0_pred[:, 0] * y2_pred[:, 0] y_pred[:, 1] = y0_pred[:, 1] * y1_pred[:, 0] y_pred[:, 2] = y0_pred[:, 1] * y1_pred[:, 1] y_pred[:, 3] = y0_pred[:, 1] * y1_pred[:, 2] y_pred[:, 4] = y0_pred[:, 0] * y2_pred[:, 1] y_pred[:, 5] = y0_pred[:, 0] * y2_pred[:, 2] y_pred[:, 6] = y0_pred[:, 0] * y2_pred[:, 3] y_pred[:, 7] = y0_pred[:, 0] * y2_pred[:, 4] y_pred[:, 8] = y0_pred[:, 0] * y2_pred[:, 5] return y_pred