class XGBCEarlyStoppingCV(XGBClassifier): def fit(self, X, y, # sample_weight=None, eval_metric=None, early_stopping_rounds=None, verbose=True, nfold=3, seed=1): xgb_options = self.get_xgb_params() self._le = LabelEncoder().fit(y) training_labels = self._le.transform(y) train_dmatrix = DMatrix(X, label=training_labels, missing=self.missing) evaluation_history = cv( xgb_options, train_dmatrix, num_boost_round=self.n_estimators, nfold=nfold, stratified=True, # folds=None, metrics=(eval_metric), # obj=None, # feval=None, # maximize=False, early_stopping_rounds=early_stopping_rounds, # fpreproc=None, # as_pandas=True, verbose_eval=verbose, show_stdv=False, seed=seed ) best_iteration = evaluation_history.index[-1] self._Booster = train( xgb_options, train_dmatrix, num_boost_round=best_iteration, verbose_eval=verbose, ) return self
def fit(self, X, y, sample_weight=None, eval_set=None, eval_metric=None, early_stopping_rounds=None, verbose=True): # pylint: disable = attribute-defined-outside-init,arguments-differ """ Fit gradient boosting classifier Parameters ---------- X : array_like Feature matrix y : array_like Labels sample_weight : array_like Weight for each instance eval_set : list, optional A list of (X, y) pairs to use as a validation set for early-stopping eval_metric : str, callable, optional If a str, should be a built-in evaluation metric to use. See doc/parameter.md. If callable, a custom evaluation metric. The call signature is func(y_predicted, y_true) where y_true will be a DMatrix object such that you may need to call the get_label method. It must return a str, value pair where the str is a name for the evaluation and value is the value of the evaluation function. This objective is always minimized. early_stopping_rounds : int, optional Activates early stopping. Validation error needs to decrease at least every <early_stopping_rounds> round(s) to continue training. Requires at least one item in evals. If there's more than one, will use the last. Returns the model from the last iteration (not the best one). If early stopping occurs, the model will have two additional fields: bst.best_score and bst.best_iteration. verbose : bool If `verbose` and an evaluation set is used, writes the evaluation metric measured on the validation set to stderr. """ evals_result = {} self.classes_ = list(np.unique(y)) self.n_classes_ = len(self.classes_) if self.n_classes_ > 2: # Switch to using a multiclass objective in the underlying XGB instance self.objective = "multi:softprob" xgb_options = self.get_xgb_params() xgb_options['num_class'] = self.n_classes_ else: xgb_options = self.get_xgb_params() feval = eval_metric if callable(eval_metric) else None if eval_metric is not None: if callable(eval_metric): eval_metric = None else: xgb_options.update({"eval_metric": eval_metric}) if eval_set is not None: # TODO: use sample_weight if given? evals = list(DMatrix(x[0], label=x[1]) for x in eval_set) nevals = len(evals) eval_names = ["validation_{}".format(i) for i in range(nevals)] evals = list(zip(evals, eval_names)) else: evals = () self._le = LabelEncoder().fit(y) training_labels = self._le.transform(y) if sample_weight is not None: train_dmatrix = DMatrix(X, label=training_labels, weight=sample_weight, missing=self.missing) else: train_dmatrix = DMatrix(X, label=training_labels, missing=self.missing) self._Booster = train(xgb_options, train_dmatrix, self.n_estimators, evals=evals, early_stopping_rounds=early_stopping_rounds, evals_result=evals_result, feval=feval, verbose_eval=verbose) if evals_result: for val in evals_result.items(): evals_result_key = list(val[1].keys())[0] evals_result[val[0]][evals_result_key] = val[1][evals_result_key] self.evals_result_ = evals_result if early_stopping_rounds is not None: self.best_score = self._Booster.best_score self.best_iteration = self._Booster.best_iteration return self
class XGBClassifier(XGBModel, XGBClassifierBase): # pylint: disable=missing-docstring,too-many-arguments,invalid-name __doc__ = """Implementation of the scikit-learn API for XGBoost classification. """ + '\n'.join(XGBModel.__doc__.split('\n')[2:]) def __init__(self, max_depth=3, learning_rate=0.1, n_estimators=100, silent=True, objective="binary:logistic", nthread=-1, gamma=0, min_child_weight=1, max_delta_step=0, subsample=1, colsample_bytree=1, colsample_bylevel=1, reg_alpha=0, reg_lambda=1, scale_pos_weight=1, base_score=0.5, seed=0, missing=None): super(XGBClassifier, self).__init__(max_depth, learning_rate, n_estimators, silent, objective, nthread, gamma, min_child_weight, max_delta_step, subsample, colsample_bytree, colsample_bylevel, reg_alpha, reg_lambda, scale_pos_weight, base_score, seed, missing) def fit(self, X, y, sample_weight=None, eval_set=None, eval_metric=None, early_stopping_rounds=None, verbose=True): # pylint: disable = attribute-defined-outside-init,arguments-differ """ Fit gradient boosting classifier Parameters ---------- X : array_like Feature matrix y : array_like Labels sample_weight : array_like Weight for each instance eval_set : list, optional A list of (X, y) pairs to use as a validation set for early-stopping eval_metric : str, callable, optional If a str, should be a built-in evaluation metric to use. See doc/parameter.md. If callable, a custom evaluation metric. The call signature is func(y_predicted, y_true) where y_true will be a DMatrix object such that you may need to call the get_label method. It must return a str, value pair where the str is a name for the evaluation and value is the value of the evaluation function. This objective is always minimized. early_stopping_rounds : int, optional Activates early stopping. Validation error needs to decrease at least every <early_stopping_rounds> round(s) to continue training. Requires at least one item in evals. If there's more than one, will use the last. Returns the model from the last iteration (not the best one). If early stopping occurs, the model will have two additional fields: bst.best_score and bst.best_iteration. verbose : bool If `verbose` and an evaluation set is used, writes the evaluation metric measured on the validation set to stderr. """ evals_result = {} self.classes_ = list(np.unique(y)) self.n_classes_ = len(self.classes_) if self.n_classes_ > 2: # Switch to using a multiclass objective in the underlying XGB instance self.objective = "multi:softprob" xgb_options = self.get_xgb_params() xgb_options['num_class'] = self.n_classes_ else: xgb_options = self.get_xgb_params() feval = eval_metric if callable(eval_metric) else None if eval_metric is not None: if callable(eval_metric): eval_metric = None else: xgb_options.update({"eval_metric": eval_metric}) if eval_set is not None: # TODO: use sample_weight if given? evals = list(DMatrix(x[0], label=x[1]) for x in eval_set) nevals = len(evals) eval_names = ["validation_{}".format(i) for i in range(nevals)] evals = list(zip(evals, eval_names)) else: evals = () self._le = LabelEncoder().fit(y) training_labels = self._le.transform(y) if sample_weight is not None: train_dmatrix = DMatrix(X, label=training_labels, weight=sample_weight, missing=self.missing) else: train_dmatrix = DMatrix(X, label=training_labels, missing=self.missing) self._Booster = train(xgb_options, train_dmatrix, self.n_estimators, evals=evals, early_stopping_rounds=early_stopping_rounds, evals_result=evals_result, feval=feval, verbose_eval=verbose) if evals_result: for val in evals_result.items(): evals_result_key = list(val[1].keys())[0] evals_result[val[0]][evals_result_key] = val[1][evals_result_key] self.evals_result_ = evals_result if early_stopping_rounds is not None: self.best_score = self._Booster.best_score self.best_iteration = self._Booster.best_iteration return self def predict(self, data, output_margin=False, ntree_limit=0): test_dmatrix = DMatrix(data, missing=self.missing) class_probs = self.booster().predict(test_dmatrix, output_margin=output_margin, ntree_limit=ntree_limit) if len(class_probs.shape) > 1: column_indexes = np.argmax(class_probs, axis=1) else: column_indexes = np.repeat(0, class_probs.shape[0]) column_indexes[class_probs > 0.5] = 1 return self._le.inverse_transform(column_indexes) def predict_proba(self, data, output_margin=False, ntree_limit=0): test_dmatrix = DMatrix(data, missing=self.missing) class_probs = self.booster().predict(test_dmatrix, output_margin=output_margin, ntree_limit=ntree_limit) if self.objective == "multi:softprob": return class_probs else: classone_probs = class_probs classzero_probs = 1.0 - classone_probs return np.vstack((classzero_probs, classone_probs)).transpose() def evals_result(self): """Return the evaluation results. If eval_set is passed to the `fit` function, you can call evals_result() to get evaluation results for all passed eval_sets. When eval_metric is also passed to the `fit` function, the evals_result will contain the eval_metrics passed to the `fit` function Returns ------- evals_result : dictionary Example ------- param_dist = {'objective':'binary:logistic', 'n_estimators':2} clf = xgb.XGBClassifier(**param_dist) clf.fit(X_train, y_train, eval_set=[(X_train, y_train), (X_test, y_test)], eval_metric='logloss', verbose=True) evals_result = clf.evals_result() The variable evals_result will contain: {'validation_0': {'logloss': ['0.604835', '0.531479']}, 'validation_1': {'logloss': ['0.41965', '0.17686']}} """ if self.evals_result_: evals_result = self.evals_result_ else: raise XGBoostError('No results.') return evals_result
def fit(self, X, y, sample_weight=None, eval_set=None, eval_metric=None, early_stopping_rounds=None, verbose=True): # pylint: disable = attribute-defined-outside-init,arguments-differ """ Fit gradient boosting classifier Parameters ---------- X : array_like Feature matrix y : array_like Labels sample_weight : array_like Weight for each instance eval_set : list, optional A list of (X, y) pairs to use as a validation set for early-stopping eval_metric : str, callable, optional If a str, should be a built-in evaluation metric to use. See doc/parameter.md. If callable, a custom evaluation metric. The call signature is func(y_predicted, y_true) where y_true will be a DMatrix object such that you may need to call the get_label method. It must return a str, value pair where the str is a name for the evaluation and value is the value of the evaluation function. This objective is always minimized. early_stopping_rounds : int, optional Activates early stopping. Validation error needs to decrease at least every <early_stopping_rounds> round(s) to continue training. Requires at least one item in evals. If there's more than one, will use the last. Returns the model from the last iteration (not the best one). If early stopping occurs, the model will have two additional fields: bst.best_score and bst.best_iteration. verbose : bool If `verbose` and an evaluation set is used, writes the evaluation metric measured on the validation set to stderr. """ evals_result = {} self.classes_ = list(np.unique(y)) self.n_classes_ = len(self.classes_) if self.n_classes_ > 2: # Switch to using a multiclass objective in the underlying XGB instance self.objective = "multi:softprob" xgb_options = self.get_xgb_params() xgb_options['num_class'] = self.n_classes_ else: xgb_options = self.get_xgb_params() feval = eval_metric if callable(eval_metric) else None if eval_metric is not None: if callable(eval_metric): eval_metric = None else: xgb_options.update({"eval_metric": eval_metric}) if eval_set is not None: # TODO: use sample_weight if given? evals = list(DMatrix(x[0], label=x[1]) for x in eval_set) nevals = len(evals) eval_names = ["validation_{}".format(i) for i in range(nevals)] evals = list(zip(evals, eval_names)) else: evals = () self._le = LabelEncoder().fit(y) training_labels = self._le.transform(y) if sample_weight is not None: train_dmatrix = DMatrix(X, label=training_labels, weight=sample_weight, missing=self.missing) else: train_dmatrix = DMatrix(X, label=training_labels, missing=self.missing) self._Booster = train(xgb_options, train_dmatrix, self.n_estimators, evals=evals, early_stopping_rounds=early_stopping_rounds, evals_result=evals_result, feval=feval, verbose_eval=verbose) if evals_result: for val in evals_result.items(): evals_result_key = list(val[1].keys())[0] evals_result[ val[0]][evals_result_key] = val[1][evals_result_key] self.evals_result_ = evals_result if early_stopping_rounds is not None: self.best_score = self._Booster.best_score self.best_iteration = self._Booster.best_iteration return self
class XGBClassifier(XGBModel, XGBClassifierBase): # pylint: disable=missing-docstring,too-many-arguments,invalid-name __doc__ = """Implementation of the scikit-learn API for XGBoost classification. """ + '\n'.join(XGBModel.__doc__.split('\n')[2:]) def __init__(self, max_depth=3, learning_rate=0.1, n_estimators=100, silent=True, objective="binary:logistic", nthread=-1, gamma=0, min_child_weight=1, max_delta_step=0, subsample=1, colsample_bytree=1, colsample_bylevel=1, reg_alpha=0, reg_lambda=1, scale_pos_weight=1, base_score=0.5, seed=0, missing=None): super(XGBClassifier, self).__init__(max_depth, learning_rate, n_estimators, silent, objective, nthread, gamma, min_child_weight, max_delta_step, subsample, colsample_bytree, colsample_bylevel, reg_alpha, reg_lambda, scale_pos_weight, base_score, seed, missing) def fit(self, X, y, sample_weight=None, eval_set=None, eval_metric=None, early_stopping_rounds=None, verbose=True): # pylint: disable = attribute-defined-outside-init,arguments-differ """ Fit gradient boosting classifier Parameters ---------- X : array_like Feature matrix y : array_like Labels sample_weight : array_like Weight for each instance eval_set : list, optional A list of (X, y) pairs to use as a validation set for early-stopping eval_metric : str, callable, optional If a str, should be a built-in evaluation metric to use. See doc/parameter.md. If callable, a custom evaluation metric. The call signature is func(y_predicted, y_true) where y_true will be a DMatrix object such that you may need to call the get_label method. It must return a str, value pair where the str is a name for the evaluation and value is the value of the evaluation function. This objective is always minimized. early_stopping_rounds : int, optional Activates early stopping. Validation error needs to decrease at least every <early_stopping_rounds> round(s) to continue training. Requires at least one item in evals. If there's more than one, will use the last. Returns the model from the last iteration (not the best one). If early stopping occurs, the model will have two additional fields: bst.best_score and bst.best_iteration. verbose : bool If `verbose` and an evaluation set is used, writes the evaluation metric measured on the validation set to stderr. """ evals_result = {} self.classes_ = list(np.unique(y)) self.n_classes_ = len(self.classes_) if self.n_classes_ > 2: # Switch to using a multiclass objective in the underlying XGB instance self.objective = "multi:softprob" xgb_options = self.get_xgb_params() xgb_options['num_class'] = self.n_classes_ else: xgb_options = self.get_xgb_params() feval = eval_metric if callable(eval_metric) else None if eval_metric is not None: if callable(eval_metric): eval_metric = None else: xgb_options.update({"eval_metric": eval_metric}) if eval_set is not None: # TODO: use sample_weight if given? evals = list(DMatrix(x[0], label=x[1]) for x in eval_set) nevals = len(evals) eval_names = ["validation_{}".format(i) for i in range(nevals)] evals = list(zip(evals, eval_names)) else: evals = () self._le = LabelEncoder().fit(y) training_labels = self._le.transform(y) if sample_weight is not None: train_dmatrix = DMatrix(X, label=training_labels, weight=sample_weight, missing=self.missing) else: train_dmatrix = DMatrix(X, label=training_labels, missing=self.missing) self._Booster = train(xgb_options, train_dmatrix, self.n_estimators, evals=evals, early_stopping_rounds=early_stopping_rounds, evals_result=evals_result, feval=feval, verbose_eval=verbose) if evals_result: for val in evals_result.items(): evals_result_key = list(val[1].keys())[0] evals_result[ val[0]][evals_result_key] = val[1][evals_result_key] self.evals_result_ = evals_result if early_stopping_rounds is not None: self.best_score = self._Booster.best_score self.best_iteration = self._Booster.best_iteration return self def predict(self, data, output_margin=False, ntree_limit=0): test_dmatrix = DMatrix(data, missing=self.missing) class_probs = self.booster().predict(test_dmatrix, output_margin=output_margin, ntree_limit=ntree_limit) if len(class_probs.shape) > 1: column_indexes = np.argmax(class_probs, axis=1) else: column_indexes = np.repeat(0, class_probs.shape[0]) column_indexes[class_probs > 0.5] = 1 return self._le.inverse_transform(column_indexes) def predict_proba(self, data, output_margin=False, ntree_limit=0): test_dmatrix = DMatrix(data, missing=self.missing) class_probs = self.booster().predict(test_dmatrix, output_margin=output_margin, ntree_limit=ntree_limit) if self.objective == "multi:softprob": return class_probs else: classone_probs = class_probs classzero_probs = 1.0 - classone_probs return np.vstack((classzero_probs, classone_probs)).transpose() def evals_result(self): """Return the evaluation results. If eval_set is passed to the `fit` function, you can call evals_result() to get evaluation results for all passed eval_sets. When eval_metric is also passed to the `fit` function, the evals_result will contain the eval_metrics passed to the `fit` function Returns ------- evals_result : dictionary Example ------- param_dist = {'objective':'binary:logistic', 'n_estimators':2} clf = xgb.XGBClassifier(**param_dist) clf.fit(X_train, y_train, eval_set=[(X_train, y_train), (X_test, y_test)], eval_metric='logloss', verbose=True) evals_result = clf.evals_result() The variable evals_result will contain: {'validation_0': {'logloss': ['0.604835', '0.531479']}, 'validation_1': {'logloss': ['0.41965', '0.17686']}} """ if self.evals_result_: evals_result = self.evals_result_ else: raise XGBoostError('No results.') return evals_result