def test_classifier_sparse_input(self): clf = RGFClassifier(prefix='clf', calc_prob='Softmax') for sparse_format in (csr_matrix, csc_matrix, coo_matrix): iris_sparse = sparse_format(self.iris.data) clf.fit(iris_sparse, self.iris.target) score = clf.score(iris_sparse, self.iris.target) self.assertGreater(score, 0.8, "Failed with score = {0:.5f}".format(score))
def test_attributes(self): clf = RGFClassifier() attributes = ('estimators_', 'classes_', 'n_classes_', 'n_features_', 'fitted_', 'sl2_', 'min_samples_leaf_', 'n_iter_') for attr in attributes: self.assertRaises(NotFittedError, getattr, clf, attr) clf.fit(self.X_train, self.y_train) self.assertEqual(len(clf.estimators_), len(np.unique(self.y_train))) np.testing.assert_array_equal(clf.classes_, sorted(np.unique(self.y_train))) self.assertEqual(clf.n_classes_, len(clf.estimators_)) self.assertEqual(clf.n_features_, self.X_train.shape[-1]) self.assertTrue(clf.fitted_) if clf.sl2 is None: self.assertEqual(clf.sl2_, clf.l2) else: self.assertEqual(clf.sl2_, clf.sl2) if clf.min_samples_leaf < 1: self.assertLessEqual(clf.min_samples_leaf_, 0.5 * self.X_train.shape[0]) else: self.assertEqual(clf.min_samples_leaf_, clf.min_samples_leaf) if clf.n_iter is None: if clf.loss == "LS": self.assertEqual(clf.n_iter_, 10) else: self.assertEqual(clf.n_iter_, 5) else: self.assertEqual(clf.n_iter_, clf.n_iter)
def test_attributes(self): clf = RGFClassifier() attributes = ('estimators_', 'classes_', 'n_classes_', 'n_features_', 'fitted_', 'sl2_', 'min_samples_leaf_', 'n_iter_') for attr in attributes: self.assertRaises(NotFittedError, getattr, clf, attr) clf.fit(self.X_train, self.y_train) self.assertEqual(len(clf.estimators_), len(np.unique(self.y_train))) np.testing.assert_array_equal(clf.classes_, sorted(np.unique(self.y_train))) self.assertEqual(clf.n_classes_, len(clf.estimators_)) self.assertEqual(clf.n_features_, self.X_train.shape[-1]) self.assertTrue(clf.fitted_) if clf.sl2 is None: self.assertEqual(clf.sl2_, clf.l2) else: self.assertEqual(clf.sl2_, clf.sl2) if clf.min_samples_leaf < 1: self.assertLessEqual(clf.min_samples_leaf_, 0.5 * self.X_train.shape[0]) else: self.assertEqual(clf.min_samples_leaf_, clf.min_samples_leaf) if clf.n_iter is None: if clf.loss == "LS": self.assertEqual(clf.n_iter_, 10) else: self.assertEqual(clf.n_iter_, 5) else: self.assertEqual(clf.n_iter_, clf.n_iter)
def test_classifier_sparse_input(self): clf = RGFClassifier(calc_prob='softmax') for sparse_format in (sparse.bsr_matrix, sparse.coo_matrix, sparse.csc_matrix, sparse.csr_matrix, sparse.dia_matrix, sparse.dok_matrix, sparse.lil_matrix): iris_sparse = sparse_format(self.iris.data) clf.fit(iris_sparse, self.iris.target) score = clf.score(iris_sparse, self.iris.target) self.assertGreater(score, 0.8, "Failed with score = {0:.5f}".format(score))
def test_softmax_classifier(self): clf = RGFClassifier(calc_prob='softmax') clf.fit(self.iris.data, self.iris.target) proba_sum = clf.predict_proba(self.iris.data).sum(axis=1) np.testing.assert_almost_equal(proba_sum, np.ones(self.iris.target.shape[0])) score = clf.score(self.iris.data, self.iris.target) print('Score: {0:.5f}'.format(score)) self.assertGreater(score, 0.8, "Failed with score = {0:.5f}".format(score))
def test_softmax_classifier(self): clf = RGFClassifier(prefix='clf', calc_prob='Softmax') clf.fit(self.iris.data, self.iris.target) proba_sum = clf.predict_proba(self.iris.data).sum(axis=1) np.testing.assert_almost_equal(proba_sum, np.ones(self.iris.target.shape[0])) score = clf.score(self.iris.data, self.iris.target) print('Score: {0:.5f}'.format(score)) self.assertGreater(score, 0.8, "Failed with score = {0:.5f}".format(score))
def test_bin_classifier(self): clf = RGFClassifier(prefix='clf') bin_target = (self.iris.target == 2).astype(int) clf.fit(self.iris.data, bin_target) proba_sum = clf.predict_proba(self.iris.data).sum(axis=1) np.testing.assert_almost_equal(proba_sum, np.ones(bin_target.shape[0])) score = clf.score(self.iris.data, bin_target) print('Score: {0:.5f}'.format(score)) self.assertGreater(score, 0.8, "Failed with score = {0:.5f}".format(score))
def test_bin_classifier(self): clf = RGFClassifier() bin_target = (self.iris.target == 2).astype(int) clf.fit(self.iris.data, bin_target) proba_sum = clf.predict_proba(self.iris.data).sum(axis=1) np.testing.assert_almost_equal(proba_sum, np.ones(bin_target.shape[0])) score = clf.score(self.iris.data, bin_target) print('Score: {0:.5f}'.format(score)) self.assertGreater(score, 0.8, "Failed with score = {0:.5f}".format(score))
def test_string_y(self): clf = RGFClassifier() y_str = np.array(self.iris.target, dtype=str) y_str[y_str == '0'] = 'Zero' y_str[y_str == '1'] = 'One' y_str[y_str == '2'] = 'Two' clf.fit(self.iris.data, y_str) y_pred = clf.predict(self.iris.data) score = accuracy_score(y_str, y_pred) self.assertGreater(score, 0.95, "Failed with score = {0:.5f}".format(score))
def test_joblib_pickle(self): clf = RGFClassifier() clf.fit(self.X_train, self.y_train) y_pred1 = clf.predict(self.X_test) joblib.dump(clf, 'test_clf.pkl') # Remove model file _cleanup() clf2 = joblib.load('test_clf.pkl') y_pred2 = clf2.predict(self.X_test) np.testing.assert_allclose(y_pred1, y_pred2)
def test_pickle(self): clf = RGFClassifier() clf.fit(self.X_train, self.y_train) y_pred1 = clf.predict(self.X_test) s = pickle.dumps(clf) # Remove model file _cleanup() reg2 = pickle.loads(s) y_pred2 = reg2.predict(self.X_test) np.testing.assert_allclose(y_pred1, y_pred2)
def test_sample_weight(self): clf = RGFClassifier() y_pred = clf.fit(self.X_train, self.y_train).predict_proba(self.X_test) y_pred_weighted = clf.fit(self.X_train, self.y_train, np.ones(self.y_train.shape[0]) ).predict_proba(self.X_test) np.testing.assert_allclose(y_pred, y_pred_weighted) weights = np.ones(self.y_train.shape[0]) * np.nextafter(np.float32(0), np.float32(1)) weights[0] = 1 y_pred_weighted = clf.fit(self.X_train, self.y_train, weights).predict(self.X_test) np.testing.assert_equal(y_pred_weighted, np.full(self.y_test.shape[0], self.y_test[0]))
def test_sample_weight(self): clf = RGFClassifier() y_pred = clf.fit(self.X_train, self.y_train).predict_proba(self.X_test) y_pred_weighted = clf.fit(self.X_train, self.y_train, np.ones(self.y_train.shape[0]) ).predict_proba(self.X_test) np.testing.assert_allclose(y_pred, y_pred_weighted) weights = np.ones(self.y_train.shape[0]) * np.nextafter(np.float32(0), np.float32(1)) weights[0] = 1 y_pred_weighted = clf.fit(self.X_train, self.y_train, weights).predict(self.X_test) np.testing.assert_equal(y_pred_weighted, np.full(self.y_test.shape[0], self.y_test[0]))
def run_rgf(): model = RGFClassifier(max_leaf=1000, algorithm="RGF", loss="Log", l2=0.01, sl2=0.01, normalize=False, min_samples_leaf=10, n_iter=None, opt_interval=100, learning_rate=.5, calc_prob="sigmoid", n_jobs=-1, memory_policy="generous", verbose=0) fit_model = model.fit(X_train, y_train) pred = fit_model.predict_proba(X_valid)[:, 1] pred_test = fit_model.predict_proba(X_test)[:, 1] try: subprocess.call('rm -rf /tmp/rgf/*', shell=True) print("Clean up is successfull") print(glob.glob("/tmp/rgf/*")) except Exception as e: print(str(e)) return pred, pred_test
def test_cleanup(self): clf1 = RGFClassifier() clf1.fit(self.X_train, self.y_train) clf2 = RGFClassifier() clf2.fit(self.X_train, self.y_train) self.assertNotEqual(clf1.cleanup(), 0) self.assertEqual(clf1.cleanup(), 0) for est in clf1.estimators_: glob_file = os.path.join(_get_temp_path(), est._file_prefix + "*") self.assertFalse(glob.glob(glob_file)) self.assertRaises(NotFittedError, clf1.predict, self.X_test) clf2.predict(self.X_test)
def run_rgf(): model = RGFClassifier( max_leaf=1000, algorithm="RGF", loss="Log", l2=0.01, sl2=0.01, normalize=False, min_samples_leaf=10, n_iter=None, opt_interval=100, learning_rate=.5, calc_prob="sigmoid", n_jobs=-1, memory_policy="generous", verbose=0 ) fit_model = model.fit( X_train, y_train ) pred = fit_model.predict_proba(X_valid)[:,1] pred_test = fit_model.predict_proba(X_test)[:,1] try: subprocess.call('rm -rf /tmp/rgf/*', shell=True) print("Clean up is successfull") print(glob.glob("/tmp/rgf/*")) except Exception as e: print(str(e)) return pred, pred_test
def test_params(self): clf = RGFClassifier() valid_params = dict(max_leaf=300, test_interval=100, algorithm='RGF_Sib', loss='Log', reg_depth=1.1, l2=0.1, sl2=None, normalize=False, min_samples_leaf=9, n_iter=None, n_tree_search=2, opt_interval=100, learning_rate=0.4, verbose=True, prefix='rgf_classifier', inc_prefix=True, calc_prob='Sigmoid', clean=True) clf.set_params(**valid_params) clf.fit(self.X_train, self.y_train) non_valid_params = dict(max_leaf=0, test_interval=0, algorithm='RGF_Test', loss=True, reg_depth=0.1, l2=11, sl2=-1.1, normalize='False', min_samples_leaf=0.7, n_iter=11.1, n_tree_search=0, opt_interval=100.1, learning_rate=-0.5, verbose=-1, prefix='', inc_prefix=1, calc_prob=True, clean=0) for key in non_valid_params: clf.set_params(**valid_params) # Reset to valid params clf.set_params(**{key: non_valid_params[key]}) # Pick and set one non-valid parametr self.assertRaises(ValueError, clf.fit, self.X_train, self.y_train)
def test_params(self): clf = RGFClassifier() valid_params = dict(max_leaf=300, test_interval=100, algorithm='RGF_Sib', loss='Log', reg_depth=1.1, l2=0.1, sl2=None, normalize=False, min_samples_leaf=9, n_iter=None, n_tree_search=2, opt_interval=100, learning_rate=0.4, calc_prob='sigmoid', n_jobs=-1, memory_policy='conservative', verbose=True) clf.set_params(**valid_params) clf.fit(self.X_train, self.y_train) non_valid_params = dict(max_leaf=0, test_interval=0, algorithm='RGF_Test', loss=True, reg_depth=0.1, l2=11, sl2=-1.1, normalize='False', min_samples_leaf=0.7, n_iter=11.1, n_tree_search=0, opt_interval=100.1, learning_rate=-0.5, calc_prob=True, n_jobs='-1', memory_policy='Generos', verbose=-1) for key in non_valid_params: clf.set_params(**valid_params) # Reset to valid params clf.set_params(**{key: non_valid_params[key]}) # Pick and set one non-valid parametr self.assertRaises(ValueError, clf.fit, self.X_train, self.y_train)
def objective(max_leaf, l2, min_samples_leaf, learning_rate): max_leaf = int(max_leaf) min_samples_leaf = int(min_samples_leaf) assert type(max_leaf) == int assert type(min_samples_leaf) == int model = RGFClassifier( max_leaf=max_leaf, l2=l2, min_samples_leaf=min_samples_leaf, learning_rate=learning_rate, algorithm="RGF_Sib", test_interval=100, ) model.fit(train_m, label_m) pred_proba = model.predict_proba(train_val) score = roc_auc_score(label_val, pred_proba[:, 1]) return score
def train(params): # log hyperparams for this run for k, v in params.items(): mlflow.log_param(k, v) # load dataset files # NOTE: to get meta data, set allow_pickle=True for np.load, then index into dataset object with key 'meta' dataset = np.load('preprocessed/dataset.npz') X_arr = dataset['X_arr'] Y_arr = dataset['Y_arr'] # split for train-test X_train, X_test, Y_train, Y_test = train_test_split(X_arr, Y_arr, stratify=Y_arr, test_size=0.2) # instantiate model with params rgf_clf = RGFClassifier(**params) rgf_clf.fit(X_train, Y_train) # predict on test data Y_pred = rgf_clf.predict(X_test) Y_pred_proba = rgf_clf.predict_proba(X_test) # log logistic loss value logistic_loss = log_loss(Y_test, Y_pred_proba) mlflow.log_metric('log_loss', logistic_loss) # log precision, recall, f1 p, r, f, _ = precision_recall_fscore_support(y_true=Y_test, y_pred=Y_pred, average='binary') mlflow.log_metric('precision', p) mlflow.log_metric('recall', r) mlflow.log_metric('f1', f) # which features matter the most print("========== FEATURE IMPORTANCES ==========") print(rgf_clf.feature_importances_)
def model_pred(trn_tmp_x,trn_tmp_y,val_tmp_x,val_tmp_y,tst_x): best_iter = 1200 model = RGFClassifier(max_leaf=best_iter, #Try increasing this as a starter algorithm="RGF", loss="Log", l2=0.01, normalize=False, min_samples_leaf=20, learning_rate=0.5, verbose=False) fit_model = model.fit( trn_tmp_x, trn_tmp_y ) return fit_model.predict_proba(val_tmp_x)[:,1], fit_model.predict_proba(tst_x)[:,1],best_iter
def rgf(df: pd.DataFrame, target: pd.DataFrame, test: pd.DataFrame, parameters: Dict): n_splits = 5 # n_neighbors = parameters["n_neighbors"] folds = KFold(n_splits=n_splits, shuffle=True, random_state=42) oof = np.zeros((df.shape[0] + test.shape[0], 9)) for trn_idx, val_idx in folds.split(df, target): train_x = df.iloc[trn_idx, :].values val_x = df.iloc[val_idx, :].values train_y = target[trn_idx].values val_y = target[val_idx].values classifier = RGFClassifier( n_jobs=14, algorithm="RGF", loss="Log", ) classifier.fit(train_x, train_y) y_hat = classifier.predict_proba(val_x) print(log_loss(val_y, y_hat)) print(oof.shape, y_hat.shape) oof[val_idx] = y_hat pred = classifier.predict_proba(test.values) oof[len(target):, :] += pred / n_splits print(oof.shape) # np.save("data/04_features/oof.npz", oof) # oof = np.load("data/04_features/oof.npy") n_name = ["knn_{}".format(i) for i in range(9)] oof = pd.DataFrame(oof) oof.to_csv("data/09_oof/rgf_{}.csv".format(3)) return oof[len(target):].values
X_test = test_df.copy() logging.info("Fold {0}".format(i)) # Enocode data for f in f_cats: X_train[f + "_avg"], X_valid[f + "_avg"], X_test[f + "_avg"] = target_encode( trn_series=X_train[f], val_series=X_valid[f], tst_series=X_test[f], target=y_train, min_samples_leaf=200, smoothing=10, noise_level=0 ) # Run model for this fold fit_model = model.fit(X_train, y_train) # Generate validation predictions for this fold pred = fit_model.predict_proba(X_valid)[:, 1] logging.info(" Gini = {0}".format(eval_gini(y_valid, pred))) y_valid_pred.iloc[test_index] = pred # Accumulate test set predictions probs = fit_model.predict_proba(X_test)[:, 1] y_test_pred += probs del X_test, X_train, X_valid, y_train y_test_pred /= K # Average test set predictions logging.info("Gini for full training set: {0}".format(eval_gini(y, y_valid_pred)))
import time from sklearn import datasets from sklearn.utils.validation import check_random_state from sklearn.ensemble import GradientBoostingClassifier from rgf.sklearn import RGFClassifier, FastRGFClassifier iris = datasets.load_iris() rng = check_random_state(0) perm = rng.permutation(iris.target.size) iris.data = iris.data[perm] iris.target = iris.target[perm] start = time.time() clf = RGFClassifier() clf.fit(iris.data, iris.target) score = clf.score(iris.data, iris.target) end = time.time() print("RGF: {} sec".format(end - start)) print("score: {}".format(score)) start = time.time() clf = FastRGFClassifier() clf.fit(iris.data, iris.target) score = clf.score(iris.data, iris.target) end = time.time() print("FastRGF: {} sec".format(end - start)) print("score: {}".format(score)) start = time.time() clf = GradientBoostingClassifier()
else: blindloodata = pd.concat([blindloodata, blindtrain]) for c in highcardinality: test['loo' + c] = ProjectOnMean(train, test, c) test.drop(highcardinality, inplace=True, axis=1) train = blindloodata train.drop(highcardinality, inplace=True, axis=1) train = train.fillna(train.mean()) test = test.fillna(train.mean()) # In[ ]: rgf = RGFClassifier( max_leaf=1000, #Try increasing this as a starter algorithm="RGF_Sib", test_interval=250, loss="Log", verbose=True) rgf.fit(train[train.columns[2:]], train.target) x = rgf.predict_proba(train[train.columns[2:]]) print(GiniScore(train.target, x[:, 1])) # In[ ]: sub = pd.read_csv('../input/sample_submission.csv') x = rgf.predict_proba(test[test.columns[2:]]) sub.target = x[:, 1] sub.to_csv('rgfsubmission.csv', index=False)
def train_predict(train_df, test_df, params, model_name=None): if model_name == None: #model_name = 'l1_rgf_%s'%datetime.now().strftime('%m%d%H%M') model_name = 'l1_rgf' log = Logger(os.path.join('log', '%s.log' % model_name)) cols = [c for c in train_df.columns if c not in ['id', 'target']] log.info('Features:') for col in cols: log.info('- %s' % col) log.info('\n') log.info('Parameters:') param_items = params.items() for param_item in param_items: log.info('- %s: %s' % (param_item[0], str(param_item[1]))) log.info('\n') X = train_df[cols].values y = train_df['target'].values X_test = test_df[cols].values prob_train = np.zeros(len(X)) prob_test = np.zeros(len(X_test)) kfold = 5 scores = [] skf = StratifiedKFold(n_splits=kfold, shuffle=True, random_state=41) for i, (train_ind, valid_ind) in enumerate(skf.split(X, y)): X_train, X_valid = X[train_ind], X[valid_ind] y_train, y_valid = y[train_ind], y[valid_ind] model = RGFClassifier(**params) model.fit(X_train, y_train) prob = model.predict_proba(X_valid)[:, 1] prob_train[valid_ind] = prob score = gini_norm(prob, y_valid) scores.append(score) log.info('- Fold %d/%d score: %f' % (i + 1, kfold, score)) prob = model.predict_proba(X_test)[:, 1] prob_test += prob / kfold try: subprocess.call('rm -rf /tmp/rgf/*', shell=True) print("Clean up is successfull") print(glob.glob("/tmp/rgf/*")) except Exception as e: print(str(e)) mean_score = np.mean(scores) log.info('- Mean score: %f' % mean_score) prob_train_df = pd.DataFrame({'id': train_df['id'], 'target': prob_train}) prob_train_df.to_csv(os.path.join('local_cv', '%s.csv.gz' % model_name), index=False, compression='gzip') prob_test_df = pd.DataFrame({'id': test_df['id'], 'target': prob_test}) prob_test_df.to_csv(os.path.join('submission', '%s.csv.gz' % model_name), index=False, compression='gzip') return mean_score
# Enocode data for f in f_cats: X_train[f + "_avg"], X_valid[f + "_avg"], X_test[f + "_avg"] = target_encode( trn_series=X_train[f], val_series=X_valid[f], tst_series=X_test[f], target=y_train, min_samples_leaf=200, smoothing=10, noise_level=0) # Run model for this fold if USE_RGF_INSTEAD: X_train = X_train.fillna(X_train.mean()) rgf.fit(X_train, y_train) elif OPTIMIZE_XGB_ROUNDS: eval_set = [(X_valid, y_valid)] fit_model = xgbmodel.fit( X_train, y_train, eval_set=eval_set, eval_metric=gini_xgb, early_stopping_rounds=XGB_EARLY_STOPPING_ROUNDS, verbose=False) print(" Best N trees = ", xgbmodel.best_ntree_limit) print(" Best gini = ", xgbmodel.best_score) elif USE_LIGHTGBM: dtrain = lgbm.Dataset(X_train, y_train) dvalid = lgbm.Dataset(X_valid, y_valid, reference=dtrain)
class Level1Model(object): train_features = [ "ps_car_13", # : 1571.65 / shadow 609.23 "ps_reg_03", # : 1408.42 / shadow 511.15 "ps_ind_05_cat", # : 1387.87 / shadow 84.72 "ps_ind_03", # : 1219.47 / shadow 230.55 "ps_ind_15", # : 922.18 / shadow 242.00 "ps_reg_02", # : 920.65 / shadow 267.50 "ps_car_14", # : 798.48 / shadow 549.58 "ps_car_12", # : 731.93 / shadow 293.62 "ps_car_01_cat", # : 698.07 / shadow 178.72 "ps_car_07_cat", # : 694.53 / shadow 36.35 "ps_ind_17_bin", # : 620.77 / shadow 23.15 "ps_car_03_cat", # : 611.73 / shadow 50.67 "ps_reg_01", # : 598.60 / shadow 178.57 "ps_car_15", # : 593.35 / shadow 226.43 "ps_ind_01", # : 547.32 / shadow 154.58 "ps_ind_16_bin", # : 475.37 / shadow 34.17 "ps_ind_07_bin", # : 435.28 / shadow 28.92 "ps_car_06_cat", # : 398.02 / shadow 212.43 "ps_car_04_cat", # : 376.87 / shadow 76.98 "ps_ind_06_bin", # : 370.97 / shadow 36.13 "ps_car_09_cat", # : 214.12 / shadow 81.38 "ps_car_02_cat", # : 203.03 / shadow 26.67 "ps_ind_02_cat", # : 189.47 / shadow 65.68 "ps_car_11", # : 173.28 / shadow 76.45 "ps_car_05_cat", # : 172.75 / shadow 62.92 "ps_calc_09", # : 169.13 / shadow 129.72 "ps_calc_05", # : 148.83 / shadow 120.68 "ps_ind_08_bin", # : 140.73 / shadow 27.63 "ps_car_08_cat", # : 120.87 / shadow 28.82 "ps_ind_09_bin", # : 113.92 / shadow 27.05 "ps_ind_04_cat", # : 107.27 / shadow 37.43 "ps_ind_18_bin", # : 77.42 / shadow 25.97 "ps_ind_12_bin", # : 39.67 / shadow 15.52 "ps_ind_14", # : 37.37 / shadow 16.65 ] combs = [ ('ps_reg_01', 'ps_car_02_cat'), ('ps_reg_01', 'ps_car_04_cat'), ] def __init__(self, strat=True, splits=5, random_state=15, submit=False, mean_sub=False, metric=None): # type: (bool, int, int, bool, bool, Callable) -> None self.curr_date = datetime.datetime.now() self._submit = submit self._id = "" self.trn = None self.target = None self.sub = None self.model = None self.metric = metric self.mean_submission = mean_sub if strat: self._folds = StratifiedKFold(n_splits=splits, shuffle=True, random_state=random_state) else: self._folds = KFold(n_splits=splits, shuffle=True, random_state=random_state) self.set_model() def set_model(self): self.model = RGFClassifier(max_leaf=1000, # 1000, algorithm="RGF", # RGF_Sib, RGF_Opt loss="Log", l2=0.01, sl2=0.01, normalize=False, min_samples_leaf=10, n_iter=None, opt_interval=100, learning_rate=.5, calc_prob="sigmoid", n_jobs=-1, memory_policy="generous", verbose=0 ) @property def do_submission(self): return self._submit @property def id(self): return self._get_id() @abc.abstractmethod def _get_id(self): self._id = "rgf_full_feat_" if self._id == "": raise ValueError("Id is not set for class " + str(type(self))) return self._id def read_data(self): self.trn = pd.read_csv("../../input/train.csv", index_col=0) self.target = self.trn["target"] del self.trn["target"] if self.do_submission: self.sub = pd.read_csv("../../input/test.csv", index_col=0) def add_combinations(self): # type: (...) -> (pd.DataFrame, Optional[DataFrame]) start = time.time() for n_c, (f1, f2) in enumerate(self.combs): name1 = f1 + "_plus_" + f2 print('current feature %60s %4d in %5.1f' % (name1, n_c + 1, (time.time() - start) / 60), end='') print('\r' * 75, end='') self.trn[name1] = self.trn[f1].apply(lambda x: str(x)) + "_" + self.trn[f2].apply(lambda x: str(x)) if self.do_submission: self.sub[name1] = self.sub[f1].apply(lambda x: str(x)) + "_" + self.sub[f2].apply(lambda x: str(x)) self.trn[name1], indexer = pd.factorize(self.trn[name1]) self.sub[name1] = indexer.get_indexer(self.sub[name1]) else: self.trn[name1], _ = pd.factorize(self.trn[name1]) def prepare_data(self): noisy_features = list(set(self.trn.columns) - set(self.train_features)) # Bin continuous variables before One-Hot Encoding for f in ["ps_reg_03", "ps_car_12", "ps_car_13", "ps_car_14"]: full_f = pd.concat([self.trn[f], self.sub[f]], axis=0) full_cut = np.array(pd.cut(full_f, 50, labels=False)) self.trn[f] = full_cut[:len(self.trn)] self.sub[f] = full_cut[len(self.trn):] del full_f del full_cut self.add_combinations() # Remove noisy features self.trn.drop(noisy_features, axis=1, inplace=True) if self.do_submission: self.sub.drop(noisy_features, axis=1, inplace=True) print(self.trn.columns) def predict_oof_and_submission(self): self.read_data() self.prepare_data() pos_ratio = .3 class_weight = {0: 1 / (2 * (1 - pos_ratio)), 1: 1 / (2 * pos_ratio)} if self.model is None: raise ValueError("Model is not set for class " + str(type(self))) if self.target is None: raise ValueError("Model is not set for class " + str(type(self))) if self.trn is None: raise ValueError("Model is not set for class " + str(type(self))) if (self.sub is None) and self.do_submission: raise ValueError("Model is not set for class " + str(type(self))) # Prepare predictors oof_preds = np.zeros(len(self.trn)) if self.sub is not None: sub_preds = np.zeros(len(self.sub)) # Go through folds start = time.time() f_cats = [f for f in self.trn.columns if "_cat" in f] for i_fold, (trn_idx, val_idx) in enumerate(self._folds.split(self.target, self.target)): # Split data trn_x, trn_y = self.trn.iloc[trn_idx].copy(), self.target.iloc[trn_idx] val_x, val_y = self.trn.iloc[val_idx].copy(), self.target.iloc[val_idx] # Compute target averages for f in f_cats: ft = TargetAverageTransformation(feature_name=f, average=TargetAverageTransformation.MEAN, min_samples_leaf=200, smoothing=10, noise_level=0) trn_x[f + "_avg"] = ft.fit_transform(data=trn_x, target=trn_y) val_x[f + "_avg"] = ft.transform(data=val_x) if self.do_submission: self.sub[f + "_avg"] = ft.transform(data=self.sub) # Fit model eval_sets = [(trn_x.values, trn_y.values), (val_x.values, val_y.values)] sample_weight = trn_y.apply(lambda x: class_weight[x]).values self.model.fit(trn_x.values, trn_y.values) # Predict OOF oof_preds[val_idx] = self.model.predict_proba(val_x.values)[:, 1] # Predict SUB if mean is requested if (self.sub is not None) and self.mean_submission: sub_preds += self.model.predict_proba(self.sub.values)[:, 1] / self._folds.n_splits # Print results of current fold print("Fold %2d score : %.6f in [%5.1f]" % (i_fold + 1, self.metric(val_y, oof_preds[val_idx]), (time.time() - start) / 60)) del trn_x del val_x gc.collect() # display OOF result oof_score = self.metric(self.target, oof_preds) print("Full OOF score : %.6f" % oof_score) # Check if we need to fit the model on the full dataset if (self.sub is not None) and not self.mean_submission: # Compute target averages for f in f_cats: ft = TargetAverageTransformation(feature_name=f, average=TargetAverageTransformation.MEAN, min_samples_leaf=200, smoothing=10, noise_level=0) self.trn[f + "_avg"] = ft.fit_transform(data=self.trn, target=self.target) self.sub[f + "_avg"] = ft.transform(data=self.sub) # Fit model self.model.fit(self.trn, self.target) # Compute prediction for submission sub_preds = self.model.predict_proba(self.sub)[:, 1] if self.do_submission: filename = "../output_preds/" + self.id filename += str(int(1e6 * oof_score)) + "_" filename += self.curr_date.strftime("%Y_%m_%d_%Hh%M") # Save OOF predictions for stacking self.trn[self.id] = oof_preds self.trn[[self.id]].to_csv(filename + "_oof.csv", float_format="%.9f") # Save submission prediction for stacking or submission self.sub["target"] = sub_preds self.sub[["target"]].to_csv(filename + "_sub.csv", float_format="%.9f")