def test_cv_lgbm_df(): X, y = make_classification_df(n_samples=1024, n_num_features=20, n_cat_features=1, class_sep=0.98, random_state=0) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0) models = [LGBMClassifier(n_estimators=300) for _ in range(5)] pred_oof, pred_test, scores, importance = cross_validate( models, X_train, y_train, X_test, cv=5, eval_func=roc_auc_score) print(scores) assert len(scores) == 5 + 1 assert scores[-1] >= 0.85 # overall roc_auc assert roc_auc_score(y_train, pred_oof) == scores[-1] assert roc_auc_score(y_test, pred_test) >= 0.85 # test roc_auc assert roc_auc_score(y_test, models[0].predict_proba(X_test) [:, 1]) >= 0.85 # make sure models are trained assert len(importance) == 5 assert list(importance[0].columns) == ['feature', 'importance'] assert len(importance[0]) == 20 + 1 assert models[0].booster_.num_trees( ) < 300 # making sure early stopping worked
def test_cv_lgbm(): X, y = make_classification(n_samples=1024, n_features=20, class_sep=0.98, random_state=0) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0) models = [LGBMClassifier(n_estimators=300) for _ in range(5)] pred_oof, pred_test, scores, importance = cross_validate( models, X_train, y_train, X_test, cv=5, eval_func=roc_auc_score, fit_params={'early_stopping_rounds': 200}) print(scores) assert len(scores) == 5 + 1 assert scores[-1] >= 0.85 # overall roc_auc assert roc_auc_score(y_train, pred_oof) == scores[-1] assert roc_auc_score(y_test, pred_test) >= 0.85 # test roc_auc assert roc_auc_score( y, models[0].predict_proba(X)[:, 1]) >= 0.85 # make sure models are trained assert len(importance) == 5 assert list(importance[0].columns) == ['feature', 'importance'] assert len(importance[0]) == 20
def test_cv_partial_evaluate(): X, y = make_classification(n_samples=1024, n_features=20, class_sep=0.98, random_state=0) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0) model = RidgeClassifier(alpha=1.0) n = 0 def _fold_count(*args): nonlocal n n += 1 cv = Take(2, KFold(5)) pred_oof, pred_test, scores, _ = cross_validate(model, X_train, y_train, X_test, cv=cv, eval_func=roc_auc_score, on_each_fold=_fold_count) assert len(scores) == 2 + 1 assert scores[-1] >= 0.8 # overall auc assert n == 2
def test_cv_sklean_binary(): X, y = make_classification(n_samples=1024, n_features=20, class_sep=0.98, random_state=0) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0) model = RidgeClassifier(alpha=1.0) pred_oof, pred_test, scores, _ = cross_validate(model, X_train, y_train, X_test, cv=5, eval_func=roc_auc_score) assert len(scores) == 5 + 1 assert scores[-1] >= 0.85 # overall auc assert roc_auc_score(y_train, pred_oof) == scores[-1] assert roc_auc_score(y_test, pred_test) >= 0.85 # test score
def test_fit_params_callback(): X, y = make_classification(n_samples=1024, n_features=20, class_sep=0.98, random_state=0) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0) models = [LGBMClassifier(n_estimators=300) for _ in range(5)] sample_weights = np.random.randint(1, 10, size=len(X_train)) sample_weights = sample_weights / sample_weights.sum() def fit_params(n: int, train_index: List[int], valid_index: List[int]): return { 'early_stopping_rounds': 100, 'sample_weight': list(sample_weights[train_index]), 'eval_sample_weight': [list(sample_weights[valid_index])] } result_w_weight = cross_validate(models, X_train, y_train, X_test, cv=5, eval_func=roc_auc_score, fit_params=fit_params) result_wo_weight = cross_validate(models, X_train, y_train, X_test, cv=5, eval_func=roc_auc_score, fit_params={'early_stopping_rounds': 50}) assert result_w_weight.scores[-1] != result_wo_weight.scores[-1]
def test_cv_sklean_regression(): X, y = make_regression(n_samples=1024, n_features=20, random_state=0) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=0) model = Ridge(alpha=1.0) pred_oof, pred_test, scores, _ = cross_validate(model, X_train, y_train, X_test, cv=5, eval_func=r2_score) print(scores) assert len(scores) == 5 + 1 assert scores[-1] >= 0.95 # overall r2 assert r2_score(y_train, pred_oof) == scores[-1] assert r2_score(y_test, pred_test) >= 0.95 # test r2
def _make_1st_stage_preds(X, y, X_test): if type_of_target(y) == 'continuous': models = [ SVR(), Ridge(random_state=0), RandomForestRegressor(n_estimators=30, random_state=0) ] else: models = [ SVC(random_state=0), LogisticRegression(random_state=0), RandomForestClassifier(n_estimators=30, random_state=0) ] results = [cross_validate(m, X, y, X_test, cv=5) for m in models] return [r.oof_prediction for r in results], [r.test_prediction for r in results]
def stacking(test_predictions: List[np.ndarray], oof_predictions: List[np.ndarray], y: pd.Series, estimator: Optional[BaseEstimator] = None, cv: Optional[Union[int, Iterable, BaseCrossValidator]] = None, groups: Optional[pd.Series] = None, type_of_target: str = 'auto', eval_func: Optional[Callable] = None) -> EnsembleResult: """ Perform stacking on predictions. Args: test_predictions: List of predicted values on test data. oof_predictions: List of predicted values on out-of-fold training data. y: Target value estimator: Estimator used for the 2nd-level model. If ``None``, the default estimator (auto-tuned linear model) will be used. cv: int, cross-validation generator or an iterable which determines the cross-validation splitting strategy. - None, to use the default ``KFold(5, random_state=0, shuffle=True)``, - integer, to specify the number of folds in a ``(Stratified)KFold``, - CV splitter (the instance of ``BaseCrossValidator``), - An iterable yielding (train, test) splits as arrays of indices. groups: Group labels for the samples. Only used in conjunction with a “Group” cv instance (e.g., ``GroupKFold``). type_of_target: The type of target variable. If ``auto``, type is inferred by ``sklearn.utils.multiclass.type_of_target``. Otherwise, ``binary``, ``continuous``, or ``multiclass`` are supported. eval_func: Evaluation metric used for calculating result score. Used only if ``oof_predictions`` and ``y`` are given. Returns: Namedtuple with following members * test_prediction: numpy array, Average prediction on test data. * oof_prediction: numpy array, Average prediction on Out-of-Fold validation data. ``None`` if ``oof_predictions`` = ``None``. * score: float, Calculated score on Out-of-Fold data. ``None`` if ``eval_func`` is ``None``. """ assert len(oof_predictions) == len( test_predictions), "Number of oof and test predictions should be same" def _stack(predictions): if predictions[0].ndim == 1: predictions = [p.reshape(len(p), -1) for p in predictions] return np.hstack(predictions) X_train = convert_input(_stack(oof_predictions)) y = convert_input_vector(y, X_train.index) X_test = convert_input(_stack(test_predictions)) assert len(X_train) == len(y) if type_of_target == 'auto': type_of_target = multiclass.type_of_target(y) if estimator is None: # if estimator is None, tuned linear estimator is used if type_of_target == 'continuous': estimator = Ridge(normalize=True, random_state=0) param_grid = { 'alpha': [0.001, 0.01, 0.1, 1, 10], } else: estimator = LogisticRegression(random_state=0) param_grid = { 'penalty': ['l1', 'l2'], 'C': [0.001, 0.01, 0.1, 1, 10], } grid_search = GridSearchCV(estimator, param_grid, cv=cv) grid_search.fit(X_train, y, groups=groups) estimator = grid_search.best_estimator_ result = cross_validate(estimator, X_train, y, X_test, cv=cv, groups=groups, eval_func=eval_func) score = result.scores[-1] if result.scores else None return EnsembleResult(result.test_prediction, result.oof_prediction, score)