def test_attributes(self): clf = RGFClassifier() attributes = ('estimators_', 'classes_', 'n_classes_', 'n_features_', 'fitted_', 'sl2_', 'min_samples_leaf_', 'n_iter_') for attr in attributes: self.assertRaises(NotFittedError, getattr, clf, attr) clf.fit(self.X_train, self.y_train) self.assertEqual(len(clf.estimators_), len(np.unique(self.y_train))) np.testing.assert_array_equal(clf.classes_, sorted(np.unique(self.y_train))) self.assertEqual(clf.n_classes_, len(clf.estimators_)) self.assertEqual(clf.n_features_, self.X_train.shape[-1]) self.assertTrue(clf.fitted_) if clf.sl2 is None: self.assertEqual(clf.sl2_, clf.l2) else: self.assertEqual(clf.sl2_, clf.sl2) if clf.min_samples_leaf < 1: self.assertLessEqual(clf.min_samples_leaf_, 0.5 * self.X_train.shape[0]) else: self.assertEqual(clf.min_samples_leaf_, clf.min_samples_leaf) if clf.n_iter is None: if clf.loss == "LS": self.assertEqual(clf.n_iter_, 10) else: self.assertEqual(clf.n_iter_, 5) else: self.assertEqual(clf.n_iter_, clf.n_iter)
def run_rgf(): model = RGFClassifier(max_leaf=1000, algorithm="RGF", loss="Log", l2=0.01, sl2=0.01, normalize=False, min_samples_leaf=10, n_iter=None, opt_interval=100, learning_rate=.5, calc_prob="sigmoid", n_jobs=-1, memory_policy="generous", verbose=0) fit_model = model.fit(X_train, y_train) pred = fit_model.predict_proba(X_valid)[:, 1] pred_test = fit_model.predict_proba(X_test)[:, 1] try: subprocess.call('rm -rf /tmp/rgf/*', shell=True) print("Clean up is successfull") print(glob.glob("/tmp/rgf/*")) except Exception as e: print(str(e)) return pred, pred_test
def test_attributes(self): clf = RGFClassifier() attributes = ('estimators_', 'classes_', 'n_classes_', 'n_features_', 'fitted_', 'sl2_', 'min_samples_leaf_', 'n_iter_') for attr in attributes: self.assertRaises(NotFittedError, getattr, clf, attr) clf.fit(self.X_train, self.y_train) self.assertEqual(len(clf.estimators_), len(np.unique(self.y_train))) np.testing.assert_array_equal(clf.classes_, sorted(np.unique(self.y_train))) self.assertEqual(clf.n_classes_, len(clf.estimators_)) self.assertEqual(clf.n_features_, self.X_train.shape[-1]) self.assertTrue(clf.fitted_) if clf.sl2 is None: self.assertEqual(clf.sl2_, clf.l2) else: self.assertEqual(clf.sl2_, clf.sl2) if clf.min_samples_leaf < 1: self.assertLessEqual(clf.min_samples_leaf_, 0.5 * self.X_train.shape[0]) else: self.assertEqual(clf.min_samples_leaf_, clf.min_samples_leaf) if clf.n_iter is None: if clf.loss == "LS": self.assertEqual(clf.n_iter_, 10) else: self.assertEqual(clf.n_iter_, 5) else: self.assertEqual(clf.n_iter_, clf.n_iter)
def test_classifier_sparse_input(self): clf = RGFClassifier(prefix='clf', calc_prob='Softmax') for sparse_format in (csr_matrix, csc_matrix, coo_matrix): iris_sparse = sparse_format(self.iris.data) clf.fit(iris_sparse, self.iris.target) score = clf.score(iris_sparse, self.iris.target) self.assertGreater(score, 0.8, "Failed with score = {0:.5f}".format(score))
def run_rgf(): model = RGFClassifier( max_leaf=1000, algorithm="RGF", loss="Log", l2=0.01, sl2=0.01, normalize=False, min_samples_leaf=10, n_iter=None, opt_interval=100, learning_rate=.5, calc_prob="sigmoid", n_jobs=-1, memory_policy="generous", verbose=0 ) fit_model = model.fit( X_train, y_train ) pred = fit_model.predict_proba(X_valid)[:,1] pred_test = fit_model.predict_proba(X_test)[:,1] try: subprocess.call('rm -rf /tmp/rgf/*', shell=True) print("Clean up is successfull") print(glob.glob("/tmp/rgf/*")) except Exception as e: print(str(e)) return pred, pred_test
def test_classifier_sparse_input(self): clf = RGFClassifier(calc_prob='softmax') for sparse_format in (sparse.bsr_matrix, sparse.coo_matrix, sparse.csc_matrix, sparse.csr_matrix, sparse.dia_matrix, sparse.dok_matrix, sparse.lil_matrix): iris_sparse = sparse_format(self.iris.data) clf.fit(iris_sparse, self.iris.target) score = clf.score(iris_sparse, self.iris.target) self.assertGreater(score, 0.8, "Failed with score = {0:.5f}".format(score))
def test_softmax_classifier(self): clf = RGFClassifier(calc_prob='softmax') clf.fit(self.iris.data, self.iris.target) proba_sum = clf.predict_proba(self.iris.data).sum(axis=1) np.testing.assert_almost_equal(proba_sum, np.ones(self.iris.target.shape[0])) score = clf.score(self.iris.data, self.iris.target) print('Score: {0:.5f}'.format(score)) self.assertGreater(score, 0.8, "Failed with score = {0:.5f}".format(score))
def test_bin_classifier(self): clf = RGFClassifier() bin_target = (self.iris.target == 2).astype(int) clf.fit(self.iris.data, bin_target) proba_sum = clf.predict_proba(self.iris.data).sum(axis=1) np.testing.assert_almost_equal(proba_sum, np.ones(bin_target.shape[0])) score = clf.score(self.iris.data, bin_target) print('Score: {0:.5f}'.format(score)) self.assertGreater(score, 0.8, "Failed with score = {0:.5f}".format(score))
def test_string_y(self): clf = RGFClassifier() y_str = np.array(self.iris.target, dtype=str) y_str[y_str == '0'] = 'Zero' y_str[y_str == '1'] = 'One' y_str[y_str == '2'] = 'Two' clf.fit(self.iris.data, y_str) y_pred = clf.predict(self.iris.data) score = accuracy_score(y_str, y_pred) self.assertGreater(score, 0.95, "Failed with score = {0:.5f}".format(score))
def test_joblib_pickle(self): clf = RGFClassifier() clf.fit(self.X_train, self.y_train) y_pred1 = clf.predict(self.X_test) joblib.dump(clf, 'test_clf.pkl') # Remove model file _cleanup() clf2 = joblib.load('test_clf.pkl') y_pred2 = clf2.predict(self.X_test) np.testing.assert_allclose(y_pred1, y_pred2)
def test_pickle(self): clf = RGFClassifier() clf.fit(self.X_train, self.y_train) y_pred1 = clf.predict(self.X_test) s = pickle.dumps(clf) # Remove model file _cleanup() reg2 = pickle.loads(s) y_pred2 = reg2.predict(self.X_test) np.testing.assert_allclose(y_pred1, y_pred2)
def model_pred(trn_tmp_x,trn_tmp_y,val_tmp_x,val_tmp_y,tst_x): best_iter = 1200 model = RGFClassifier(max_leaf=best_iter, #Try increasing this as a starter algorithm="RGF", loss="Log", l2=0.01, normalize=False, min_samples_leaf=20, learning_rate=0.5, verbose=False) fit_model = model.fit( trn_tmp_x, trn_tmp_y ) return fit_model.predict_proba(val_tmp_x)[:,1], fit_model.predict_proba(tst_x)[:,1],best_iter
def test_sample_weight(self): clf = RGFClassifier() y_pred = clf.fit(self.X_train, self.y_train).predict_proba(self.X_test) y_pred_weighted = clf.fit(self.X_train, self.y_train, np.ones(self.y_train.shape[0]) ).predict_proba(self.X_test) np.testing.assert_allclose(y_pred, y_pred_weighted) weights = np.ones(self.y_train.shape[0]) * np.nextafter(np.float32(0), np.float32(1)) weights[0] = 1 y_pred_weighted = clf.fit(self.X_train, self.y_train, weights).predict(self.X_test) np.testing.assert_equal(y_pred_weighted, np.full(self.y_test.shape[0], self.y_test[0]))
def test_sample_weight(self): clf = RGFClassifier() y_pred = clf.fit(self.X_train, self.y_train).predict_proba(self.X_test) y_pred_weighted = clf.fit(self.X_train, self.y_train, np.ones(self.y_train.shape[0]) ).predict_proba(self.X_test) np.testing.assert_allclose(y_pred, y_pred_weighted) weights = np.ones(self.y_train.shape[0]) * np.nextafter(np.float32(0), np.float32(1)) weights[0] = 1 y_pred_weighted = clf.fit(self.X_train, self.y_train, weights).predict(self.X_test) np.testing.assert_equal(y_pred_weighted, np.full(self.y_test.shape[0], self.y_test[0]))
def rgf04(x_train, y_train, x_test, folds, max_round, n_splits=5): clf = RGFClassifier( max_leaf=1000, algorithm="RGF", loss="Log", l2=0.01, sl2=0.01, normalize=False, min_samples_leaf=7, # 10, n_iter=None, opt_interval=100, learning_rate=.45, # .3, calc_prob="sigmoid", n_jobs=-2, memory_policy="generous", verbose=0) # Additional processing of data x_train, x_test = feature_engineering_4(x_train, x_test, y_train) # Cross Validate cv = Cross_Validate(rgf04.__name__, n_splits, x_train.shape[0], x_test.shape[0], clf, -1, -1) cv.cross_validate(x_train, y_train, x_test, folds, verbose_eval=True) return cv.trn_gini, cv.y_trn, cv.y_tst, cv.fscore
def set_model(self): self.model = RGFClassifier(max_leaf=1000, # 1000, algorithm="RGF", # RGF_Sib, RGF_Opt loss="Log", l2=0.01, sl2=0.01, normalize=False, min_samples_leaf=10, n_iter=None, opt_interval=100, learning_rate=.5, calc_prob="sigmoid", n_jobs=-1, memory_policy="generous", verbose=0 )
def test_parallel_gridsearch(self): param_grid = dict(max_leaf=[100, 300]) grid = GridSearchCV(RGFClassifier(n_jobs=1), param_grid=param_grid, refit=True, cv=2, verbose=0, n_jobs=-1) grid.fit(self.X_train, self.y_train) y_pred = grid.best_estimator_.predict(self.X_train) score = accuracy_score(self.y_train, y_pred) self.assertGreater(score, 0.95, "Failed with score = {0:.5f}".format(score))
def test_params(self): clf = RGFClassifier() valid_params = dict(max_leaf=300, test_interval=100, algorithm='RGF_Sib', loss='Log', reg_depth=1.1, l2=0.1, sl2=None, normalize=False, min_samples_leaf=9, n_iter=None, n_tree_search=2, opt_interval=100, learning_rate=0.4, verbose=True, prefix='rgf_classifier', inc_prefix=True, calc_prob='Sigmoid', clean=True) clf.set_params(**valid_params) clf.fit(self.X_train, self.y_train) non_valid_params = dict(max_leaf=0, test_interval=0, algorithm='RGF_Test', loss=True, reg_depth=0.1, l2=11, sl2=-1.1, normalize='False', min_samples_leaf=0.7, n_iter=11.1, n_tree_search=0, opt_interval=100.1, learning_rate=-0.5, verbose=-1, prefix='', inc_prefix=1, calc_prob=True, clean=0) for key in non_valid_params: clf.set_params(**valid_params) # Reset to valid params clf.set_params(**{key: non_valid_params[key]}) # Pick and set one non-valid parametr self.assertRaises(ValueError, clf.fit, self.X_train, self.y_train)
def model_selection(select, X): if select == 'xgb': filename = "main_xgboost" # xgboost model model = XGBClassifier(num_round=1000, nthread=25, eta=0.02, gamma=1, max_depth=20, min_child_weight=0.1, subsample=0.9, colsample_bytree=0.5, objective='binary:logistic', seed=1) elif select == 'rgf': filename = "main_rgf" # regression greedy forest model model = RGFClassifier(max_leaf=400, algorithm="RGF", test_interval=150, loss="LS") elif select == 'logit': filename = "main_logit" # LogisticRegression model model = LogisticRegression(C=0.7, penalty="l2") elif select == 'knn': filename = "main_knn" # KNearestNeighbor model model = KNeighborsClassifier(n_neighbors=7) elif select == 'xtree': filename = "main_xtratree" # extratree model model = ExtraTreesClassifier(n_estimators=10000, criterion='entropy', max_depth=9, min_samples_leaf=1, n_jobs=30, random_state=1) elif select == 'rfc': filename = "main_rfc" # RandomForest model model = RandomForestClassifier(random_state=13) elif select == 'cat': filename = "main_catboost" # Catboost model model = CatBoostClassifier(iterations=80, depth=3, learning_rate=0.1, loss_function='Logloss') elif select == 'svm': filename = "main_svm" # Support vector machine model model = svm.SVC(kernel='linear', probability=True) elif select == 'kerasnn': filename = "main_kerasnn" # Keras Neural network model model = keras_network(X) else: filename = "main_lgmboost" # light gradient boosting model model = lgb.LGBMClassifier(num_leaves=150, objective='binary', max_depth=6, learning_rate=.01, max_bin=400, auc='binary_logloss') return filename, model
def objective(max_leaf, l2, min_samples_leaf, learning_rate): max_leaf = int(max_leaf) min_samples_leaf = int(min_samples_leaf) assert type(max_leaf) == int assert type(min_samples_leaf) == int model = RGFClassifier( max_leaf=max_leaf, l2=l2, min_samples_leaf=min_samples_leaf, learning_rate=learning_rate, algorithm="RGF_Sib", test_interval=100, ) model.fit(train_m, label_m) pred_proba = model.predict_proba(train_val) score = roc_auc_score(label_val, pred_proba[:, 1]) return score
def test_input_arrays_shape(self): clf = RGFClassifier() n_samples = self.y_train.shape[0] self.assertRaises(ValueError, clf.fit, self.X_train, self.y_train[:(n_samples - 1)]) self.assertRaises(ValueError, clf.fit, self.X_train, self.y_train, np.ones(n_samples - 1)) self.assertRaises(ValueError, clf.fit, self.X_train, self.y_train, np.ones((n_samples, 2)))
def model(opt): rgf = RGFClassifier( max_leaf=opt["max_leaf"], reg_depth=opt["reg_depth"], min_samples_leaf=opt["min_samples_leaf"], algorithm="RGF_Sib", test_interval=100, verbose=False, ) scores = cross_val_score(rgf, X, y, cv=3) return scores.mean()
def test_params(self): clf = RGFClassifier() valid_params = dict(max_leaf=300, test_interval=100, algorithm='RGF_Sib', loss='Log', reg_depth=1.1, l2=0.1, sl2=None, normalize=False, min_samples_leaf=9, n_iter=None, n_tree_search=2, opt_interval=100, learning_rate=0.4, calc_prob='sigmoid', n_jobs=-1, memory_policy='conservative', verbose=True) clf.set_params(**valid_params) clf.fit(self.X_train, self.y_train) non_valid_params = dict(max_leaf=0, test_interval=0, algorithm='RGF_Test', loss=True, reg_depth=0.1, l2=11, sl2=-1.1, normalize='False', min_samples_leaf=0.7, n_iter=11.1, n_tree_search=0, opt_interval=100.1, learning_rate=-0.5, calc_prob=True, n_jobs='-1', memory_policy='Generos', verbose=-1) for key in non_valid_params: clf.set_params(**valid_params) # Reset to valid params clf.set_params(**{key: non_valid_params[key]}) # Pick and set one non-valid parametr self.assertRaises(ValueError, clf.fit, self.X_train, self.y_train)
def test_softmax_classifier(self): clf = RGFClassifier(prefix='clf', calc_prob='Softmax') clf.fit(self.iris.data, self.iris.target) proba_sum = clf.predict_proba(self.iris.data).sum(axis=1) np.testing.assert_almost_equal(proba_sum, np.ones(self.iris.target.shape[0])) score = clf.score(self.iris.data, self.iris.target) print('Score: {0:.5f}'.format(score)) self.assertGreater(score, 0.8, "Failed with score = {0:.5f}".format(score))
def test_bin_classifier(self): clf = RGFClassifier(prefix='clf') bin_target = (self.iris.target == 2).astype(int) clf.fit(self.iris.data, bin_target) proba_sum = clf.predict_proba(self.iris.data).sum(axis=1) np.testing.assert_almost_equal(proba_sum, np.ones(bin_target.shape[0])) score = clf.score(self.iris.data, bin_target) print('Score: {0:.5f}'.format(score)) self.assertGreater(score, 0.8, "Failed with score = {0:.5f}".format(score))
def __init__(self, task, fast=False): if task == 'classification': self.metric = 'roc_auc' self.task = "classification" if fast: self.model = FastRGFClassifier() else: self.model = RGFClassifier(loss="Log") else: self.metric = 'neg_mean_squared_error' self.task = "regression" if fast: self.model = FastRGFRegressor() else: self.model = RGFRegressor(loss="LS", normalize=True) self.X_test = None self.X_train = None self.y_test = None self.y_train = None self.grid_search = None self.y_predict = None self.test_score = None
def train(params): # log hyperparams for this run for k, v in params.items(): mlflow.log_param(k, v) # load dataset files # NOTE: to get meta data, set allow_pickle=True for np.load, then index into dataset object with key 'meta' dataset = np.load('preprocessed/dataset.npz') X_arr = dataset['X_arr'] Y_arr = dataset['Y_arr'] # split for train-test X_train, X_test, Y_train, Y_test = train_test_split(X_arr, Y_arr, stratify=Y_arr, test_size=0.2) # instantiate model with params rgf_clf = RGFClassifier(**params) rgf_clf.fit(X_train, Y_train) # predict on test data Y_pred = rgf_clf.predict(X_test) Y_pred_proba = rgf_clf.predict_proba(X_test) # log logistic loss value logistic_loss = log_loss(Y_test, Y_pred_proba) mlflow.log_metric('log_loss', logistic_loss) # log precision, recall, f1 p, r, f, _ = precision_recall_fscore_support(y_true=Y_test, y_pred=Y_pred, average='binary') mlflow.log_metric('precision', p) mlflow.log_metric('recall', r) mlflow.log_metric('f1', f) # which features matter the most print("========== FEATURE IMPORTANCES ==========") print(rgf_clf.feature_importances_)
def rgf(df: pd.DataFrame, target: pd.DataFrame, test: pd.DataFrame, parameters: Dict): n_splits = 5 # n_neighbors = parameters["n_neighbors"] folds = KFold(n_splits=n_splits, shuffle=True, random_state=42) oof = np.zeros((df.shape[0] + test.shape[0], 9)) for trn_idx, val_idx in folds.split(df, target): train_x = df.iloc[trn_idx, :].values val_x = df.iloc[val_idx, :].values train_y = target[trn_idx].values val_y = target[val_idx].values classifier = RGFClassifier( n_jobs=14, algorithm="RGF", loss="Log", ) classifier.fit(train_x, train_y) y_hat = classifier.predict_proba(val_x) print(log_loss(val_y, y_hat)) print(oof.shape, y_hat.shape) oof[val_idx] = y_hat pred = classifier.predict_proba(test.values) oof[len(target):, :] += pred / n_splits print(oof.shape) # np.save("data/04_features/oof.npz", oof) # oof = np.load("data/04_features/oof.npy") n_name = ["knn_{}".format(i) for i in range(9)] oof = pd.DataFrame(oof) oof.to_csv("data/09_oof/rgf_{}.csv".format(3)) return oof[len(target):].values
from sklearn import datasets from sklearn.utils.validation import check_random_state from sklearn.model_selection import StratifiedKFold, cross_val_score from sklearn.ensemble import GradientBoostingClassifier from rgf.sklearn import RGFClassifier iris = datasets.load_iris() rng = check_random_state(0) perm = rng.permutation(iris.target.size) iris.data = iris.data[perm] iris.target = iris.target[perm] rgf = RGFClassifier(max_leaf=400, algorithm="RGF_Sib", test_interval=100, verbose=True) gb = GradientBoostingClassifier(n_estimators=20, learning_rate=0.01, subsample=0.6, random_state=rng) n_folds = 3 rgf_scores = cross_val_score(rgf, iris.data, iris.target, cv=StratifiedKFold(n_folds)) gb_scores = cross_val_score(gb, iris.data, iris.target,
if __name__ == '__main__': print(sys.argv) if len(sys.argv) >= 2: model = model_set[sys.argv[1]] else: model = model_set['xgb1'] metric = model['metric'] if model['mdl_type'] == 'xgb': mdl = XGBClassifier(**model['param']) elif model['mdl_type'] == 'lgb': mdl = LGBMClassifier(**model['param']) elif model['mdl_type'] == 'rgf': mdl = RGFClassifier(**model['param']) elif model['mdl_type'] == 'lr': mdl = LogisticRegression(**model['param']) elif model['mdl_type'] == 'xgbl': mdl = XGBRegressor(**model['param']) elif model['mdl_type'] == 'mlp': mdl = MLPClassifier(**model['param']) train_pred, test_pred, mean, std, full_score = five_fold_with_baging( l2_train.values, y, l2_test.values, train_id, test_id, metric, mdl,
subsample=.8, min_child_weight=6, colsample_bytree=.8, scale_pos_weight=1.6, gamma=10, reg_alpha=8, reg_lambda=1.3, ) rgf = RGFClassifier( # See https://www.kaggle.com/scirpus/regularized-greedy-forest#241285 max_leaf=1200, # Parameters suggested by olivier in link above algorithm="RGF", loss="Log", l2=0.01, sl2=0.01, normalize=False, min_samples_leaf=10, n_iter=None, opt_interval=100, learning_rate=.5, calc_prob="sigmoid", n_jobs=-1, memory_policy="generous", verbose=0) gini_results = [] # Run CV for i, (train_index, test_index) in enumerate(kf.split(train_df)): # Create data for this fold y_train, y_valid = y.iloc[train_index].copy(), y.iloc[test_index]
def train_classifiers(X_data, y): """ Trains several classifiers and reporting model quality. :param X_data: :param y: :return: trained models """ # Split the dataset into Train and Test seed = 7 test_size = 0.3 X_train, X_test, y_train, y_test = train_test_split(X_data, y, test_size=test_size, random_state=seed) svm = SVC() svm_params = { 'C': [1, 10, 100, 1000], 'gamma': [1, 0.1, 0.001, 0.0001], 'kernel': ['linear', 'rbf'] } svm_model, svm_grid = train_single_classifier_type(svm, "SVM", svm_params, X_train, X_test, y_train, y_test) knn = KNeighborsClassifier() knn_params = { 'n_neighbors': [5, 6, 7, 8, 9, 10], 'leaf_size': [1, 2, 3, 5], 'weights': ['uniform', 'distance'], 'algorithm': ['auto', 'ball_tree', 'kd_tree', 'brute'], 'n_jobs': [-1] } knn_model, knn_grid = train_single_classifier_type(knn, "KNN", knn_params, X_train, X_test, y_train, y_test) # Train the XGboost Model for Classification xgb_model = xgb.XGBClassifier() # brute force scan for all parameters, here are the tricks # usually max_depth is 6,7,8 # learning rate is around 0.05, but small changes may make big diff # tuning min_child_weight subsample colsample_bytree can have # much fun of fighting against overfit # n_estimators is how many round of boosting # finally, ensemble xgboost with multiple seeds may reduce variance xgb_parameters = { 'nthread': [4], # when use hyperthread, xgboost may become slower 'objective': ['binary:logistic'], 'learning_rate': [0.05, 0.1], # so called `eta` value 'max_depth': [6, 7, 8], 'min_child_weight': [1, 11], 'silent': [1], 'subsample': [0.8], 'colsample_bytree': [0.7, 0.8], 'n_estimators': [5, 100, 1000], # number of trees, change it to 1000 for better results 'missing': [-999], 'seed': [1337] } train_model1, xgb_grid = train_single_classifier_type( xgb_model, "XGBoost", xgb_parameters, X_train, X_test, y_train, y_test) rfc = RandomForestClassifier() rfc_parameters = { 'max_depth': [4, 5, 6], 'n_estimators': [100, 200], 'criterion': ['gini', 'entropy'], 'max_features': ['auto', 'sqrt', 'log2'], 'min_samples_leaf': [2, 4], 'min_samples_split': [2, 5, 10], } rfc_model, rfc_grid = train_single_classifier_type(rfc, "Random Forest", rfc_parameters, X_train, X_test, y_train, y_test) ext = ExtraTreesClassifier() ext_parameters = { 'n_estimators': [50, 100], 'max_features': [5, 10, 25], 'min_samples_leaf': [2, 5, 10], 'min_samples_split': [2, 5, 10], } ext_model, ext_grid = train_single_classifier_type(ext, "Extra Trees", ext_parameters, X_train, X_test, y_train, y_test) lgbm = LGBMClassifier( boosting_type='gbdt', objective='binary', n_jobs=-1, # Updated from 'nthread' silent=True) # Create parameters to search lgbm_parameters = { 'max_depth': [5, 6, 7, 8, 9, 10, 15, 20], 'learning_rate': [0.005], 'n_estimators': [100, 150, 500], 'num_leaves': [6, 8, 12, 16], 'boosting_type': ['gbdt'], 'objective': ['binary'], 'random_state': [501], # Updated from 'seed' 'colsample_bytree': [0.65], 'subsample': [0.7], 'reg_alpha': [1, 10], 'reg_lambda': [10, 100], } lgbm_model, lgbm_grid = train_single_classifier_type( lgbm, "LGBM", lgbm_parameters, X_train, X_test, y_train, y_test) rgf = RGFClassifier() rgf_parameters = { 'max_leaf': [900], 'l2': [0.1, 0.05, 1.0], 'min_samples_leaf': [5, 4, 3], 'algorithm': ["RGF", "RGF_Opt", "RGF_Sib"], 'loss': ["Log"], } rgf_model, rgf_grid = train_single_classifier_type(rgf, "RGF", rgf_parameters, X_train, X_test, y_train, y_test) frgf = FastRGFClassifier() frgf_parameters = { 'max_leaf': [100, 200, 900], 'n_estimators': [100, 1000], 'max_bin': [10, 100], 'l2': [0.1, 100, 1000], 'min_samples_leaf': [5, 6], 'opt_algorithm': ['rgf'], 'loss': ["LS"], } frgf_model, frgf_grid = train_single_classifier_type( frgf, "FRGF", frgf_parameters, X_train, X_test, y_train, y_test) return svm_model, svm_grid, \ train_model1, xgb_grid, \ rfc_model, rfc_grid, \ ext_model, ext_grid, \ lgbm_model, lgbm_grid, \ rgf_model, rgf_grid, \ frgf_model, frgf_grid
def train_predict(train_df, test_df, params, model_name=None): if model_name == None: #model_name = 'l1_rgf_%s'%datetime.now().strftime('%m%d%H%M') model_name = 'l1_rgf' log = Logger(os.path.join('log', '%s.log' % model_name)) cols = [c for c in train_df.columns if c not in ['id', 'target']] log.info('Features:') for col in cols: log.info('- %s' % col) log.info('\n') log.info('Parameters:') param_items = params.items() for param_item in param_items: log.info('- %s: %s' % (param_item[0], str(param_item[1]))) log.info('\n') X = train_df[cols].values y = train_df['target'].values X_test = test_df[cols].values prob_train = np.zeros(len(X)) prob_test = np.zeros(len(X_test)) kfold = 5 scores = [] skf = StratifiedKFold(n_splits=kfold, shuffle=True, random_state=41) for i, (train_ind, valid_ind) in enumerate(skf.split(X, y)): X_train, X_valid = X[train_ind], X[valid_ind] y_train, y_valid = y[train_ind], y[valid_ind] model = RGFClassifier(**params) model.fit(X_train, y_train) prob = model.predict_proba(X_valid)[:, 1] prob_train[valid_ind] = prob score = gini_norm(prob, y_valid) scores.append(score) log.info('- Fold %d/%d score: %f' % (i + 1, kfold, score)) prob = model.predict_proba(X_test)[:, 1] prob_test += prob / kfold try: subprocess.call('rm -rf /tmp/rgf/*', shell=True) print("Clean up is successfull") print(glob.glob("/tmp/rgf/*")) except Exception as e: print(str(e)) mean_score = np.mean(scores) log.info('- Mean score: %f' % mean_score) prob_train_df = pd.DataFrame({'id': train_df['id'], 'target': prob_train}) prob_train_df.to_csv(os.path.join('local_cv', '%s.csv.gz' % model_name), index=False, compression='gzip') prob_test_df = pd.DataFrame({'id': test_df['id'], 'target': prob_test}) prob_test_df.to_csv(os.path.join('submission', '%s.csv.gz' % model_name), index=False, compression='gzip') return mean_score
# Set up folds K = Number_of_folds kf = comm_skf sl2_list = [0.08, 0.09, 0.11, 0.12] for sl2 in sl2_list: logging.info('test with sl2 : {0}'.format(sl2)) # Set up classifier model = RGFClassifier( max_leaf=1200, algorithm="RGF", loss="Log", l2=0.012, sl2=sl2, normalize=False, min_samples_leaf=10, n_iter=None, opt_interval=100, learning_rate=0.5, calc_prob="sigmoid", n_jobs=-1, memory_policy="generous", verbose=0 ) # Run CV logging.info('feature shape {0}'.format(X.shape)) for i, (train_index, test_index) in enumerate(kf.split(train_df, y)): # Create data for this fold y_train, y_valid = y.iloc[train_index].copy(), y.iloc[test_index]
import time from sklearn import datasets from sklearn.utils.validation import check_random_state from sklearn.ensemble import GradientBoostingClassifier from rgf.sklearn import RGFClassifier, FastRGFClassifier iris = datasets.load_iris() rng = check_random_state(0) perm = rng.permutation(iris.target.size) iris.data = iris.data[perm] iris.target = iris.target[perm] start = time.time() clf = RGFClassifier() clf.fit(iris.data, iris.target) score = clf.score(iris.data, iris.target) end = time.time() print("RGF: {} sec".format(end - start)) print("score: {}".format(score)) start = time.time() clf = FastRGFClassifier() clf.fit(iris.data, iris.target) score = clf.score(iris.data, iris.target) end = time.time() print("FastRGF: {} sec".format(end - start)) print("score: {}".format(score)) start = time.time() clf = GradientBoostingClassifier()
xgb_homeless_bag_kf = layer1.Layer1Train(XGBClassifier(**xgb_homeless_params), 'xgb_homeless_bag_kf', drop_stupid='default', cat_transform='smooth', recon_category=True, engineer_stats=True) xgb_18 = layer1.Layer1Train(XGBClassifier(**xgb_18_params), 'xgb_18', drop_stupid='default', cat_transform='smooth', recon_category=True, data_transform='round', engineer_stats=True) rgf_bojan_na_bag_kf = layer1.Layer1Train(RGFClassifier(**rgf_bojan_params), 'rgf_bojan_na_bag_kf', drop_stupid='default', cat_transform='smooth', recon_category=True, engineer_stats=True) # fm_sgd = layer1.Layer1Train(sgd.FMClassification(**fm_sgd_params), 'fm_sgd', # drop_stupid=True, cat_transform='onehot', recon_category=True, engineer_stats=True) # # fm_als = layer1.Layer1Train(sgd.FMClassification(**fm_als_params), 'fm_als', # drop_stupid=True, cat_transform='smooth', data_transform='log', recon_category=True, engineer_stats=True) cat_1 = layer1.Layer1Train(CatBoostClassifier(**cat_1_params), 'cat_1', drop_stupid='default',
def test_cleanup(self): clf1 = RGFClassifier() clf1.fit(self.X_train, self.y_train) clf2 = RGFClassifier() clf2.fit(self.X_train, self.y_train) self.assertNotEqual(clf1.cleanup(), 0) self.assertEqual(clf1.cleanup(), 0) for est in clf1.estimators_: glob_file = os.path.join(_get_temp_path(), est._file_prefix + "*") self.assertFalse(glob.glob(glob_file)) self.assertRaises(NotFittedError, clf1.predict, self.X_test) clf2.predict(self.X_test)