def score(self, X, y, objectives):
"""Evaluate model performance on current and additional objectives.
Arguments:
X (ww.DataTable, pd.DataFrame or np.ndarray): Data of shape [n_samples, n_features]
y (ww.DataColumn, pd.Series): True labels of length [n_samples]
objectives (list): Non-empty list of objectives to score on
Returns:
dict: Ordered dictionary of objective scores
"""
X, y = self._convert_to_woodwork(X, y)
X = _convert_woodwork_types_wrapper(X.to_dataframe())
y = _convert_woodwork_types_wrapper(y.to_series())
objectives = [get_objective(o, return_instance=True) for o in objectives]
y_encoded = self._encode_targets(y)
y_shifted = y_encoded.shift(-self.gap)
y_predicted, y_predicted_proba = self._compute_predictions(X, y, objectives, time_series=True)
if y_predicted is not None:
y_predicted = _convert_woodwork_types_wrapper(y_predicted.to_series())
if y_predicted_proba is not None:
y_predicted_proba = _convert_woodwork_types_wrapper(y_predicted_proba.to_dataframe())
y_shifted, y_predicted, y_predicted_proba = drop_rows_with_nans(y_shifted, y_predicted, y_predicted_proba)
return self._score_all_objectives(X, y_shifted, y_predicted,
y_pred_proba=y_predicted_proba,
objectives=objectives)
def _predict(self, X, y, objective=None, pad=False):
features = self.compute_estimator_features(X, y)
features_no_nan, y_no_nan = drop_rows_with_nans(features, y)
if objective is not None:
objective = get_objective(objective, return_instance=True)
if not objective.is_defined_for_problem_type(self.problem_type):
raise ValueError(
f"Objective {objective.name} is not defined for time series binary classification."
)
if self.threshold is None:
predictions = self._estimator_predict(features_no_nan, y_no_nan)
else:
proba = self._estimator_predict_proba(features_no_nan, y_no_nan)
proba = proba.iloc[:, 1]
if objective is None:
predictions = proba > self.threshold
else:
predictions = objective.decision_function(
proba, threshold=self.threshold, X=features_no_nan)
if pad:
return pad_with_nans(
predictions, max(0, features.shape[0] - predictions.shape[0]))
return predictions
def score(self, X, y, objectives):
"""Evaluate model performance on current and additional objectives.
Arguments:
X (ww.DataTable, pd.DataFrame or np.ndarray): Data of shape [n_samples, n_features]
y (pd.Series, ww.DataColumn): True labels of length [n_samples]
objectives (list): Non-empty list of objectives to score on
Returns:
dict: Ordered dictionary of objective scores
"""
# Only converting X for the call to _score_all_objectives
if X is None:
X = pd.DataFrame()
X = _convert_to_woodwork_structure(X)
X = _convert_woodwork_types_wrapper(X.to_dataframe())
y = _convert_to_woodwork_structure(y)
y = _convert_woodwork_types_wrapper(y.to_series())
y_predicted = self.predict(X, y)
y_shifted = y.shift(-self.gap)
objectives = [
get_objective(o, return_instance=True) for o in objectives
]
y_shifted, y_predicted = drop_rows_with_nans(y_shifted, y_predicted)
return self._score_all_objectives(X,
y_shifted,
y_predicted,
y_pred_proba=None,
objectives=objectives)
def predict(self, X, y=None, objective=None):
"""Make predictions using selected features.
Arguments:
X (ww.DataTable, pd.DataFrame, or np.ndarray): Data of shape [n_samples, n_features]
y (ww.DataColumn, pd.Series, np.ndarray, None): The target training targets of length [n_samples]
objective (Object or string): The objective to use to make predictions
Returns:
pd.Series: Predicted values.
"""
if X is None:
X = pd.DataFrame()
X = _convert_to_woodwork_structure(X)
y = _convert_to_woodwork_structure(y)
X = _convert_woodwork_types_wrapper(X.to_dataframe())
y = _convert_woodwork_types_wrapper(y.to_series())
features = self.compute_estimator_features(X, y)
features_no_nan, y = drop_rows_with_nans(features, y)
y_arg = None
if self.estimator.predict_uses_y:
y_arg = y
predictions = self.estimator.predict(features_no_nan, y_arg)
predictions = predictions.rename(self.input_target_name)
return pad_with_nans(predictions,
max(0, features.shape[0] - predictions.shape[0]))
def _predict(self, X, y, objective=None, pad=False):
features = self.compute_estimator_features(X, y)
features = _convert_woodwork_types_wrapper(features.to_dataframe())
features_no_nan, y_no_nan = drop_rows_with_nans(features, y)
predictions = self._estimator_predict(features_no_nan, y_no_nan)
if pad:
padded = pad_with_nans(predictions.to_series(), max(0, features.shape[0] - predictions.shape[0]))
return _convert_to_woodwork_structure(padded)
return predictions
def _predict(self, X, y, objective=None, pad=False):
features = self.compute_estimator_features(X, y)
features_no_nan, y_no_nan = drop_rows_with_nans(features, y)
predictions = self._estimator_predict(features_no_nan, y_no_nan)
if pad:
return pad_with_nans(
predictions, max(0, features.shape[0] - predictions.shape[0]))
return predictions
def predict_proba(self, X, y=None):
"""Make probability estimates for labels.
Arguments:
X (ww.DataTable, pd.DataFrame or np.ndarray): Data of shape [n_samples, n_features]
Returns:
pd.DataFrame: Probability estimates
"""
X, y = self._convert_to_woodwork(X, y)
X = _convert_woodwork_types_wrapper(X.to_dataframe())
y = _convert_woodwork_types_wrapper(y.to_series())
features = self.compute_estimator_features(X, y)
features_no_nan, y_no_nan = drop_rows_with_nans(features, y)
proba = self._estimator_predict_proba(features_no_nan, y_no_nan)
proba.columns = self._encoder.classes_
return pad_with_nans(proba, max(0, features.shape[0] - proba.shape[0]))
def fit(self, X, y):
"""Fit a time series classification pipeline.
Arguments:
X (ww.DataTable, pd.DataFrame or np.ndarray): The input training data of shape [n_samples, n_features]
y (ww.DataColumn, pd.Series, np.ndarray): The target training targets of length [n_samples]
Returns:
self
"""
X, y = self._convert_to_woodwork(X, y)
X = _convert_woodwork_types_wrapper(X.to_dataframe())
y = _convert_woodwork_types_wrapper(y.to_series())
self._encoder.fit(y)
y = self._encode_targets(y)
X_t = self._compute_features_during_fit(X, y)
X_t = _convert_woodwork_types_wrapper(X_t.to_dataframe())
y_shifted = y.shift(-self.gap)
X_t, y_shifted = drop_rows_with_nans(X_t, y_shifted)
self.estimator.fit(X_t, y_shifted)
return self
def explain_predictions_best_worst(pipeline,
input_features,
y_true,
num_to_explain=5,
top_k_features=3,
include_shap_values=False,
metric=None,
output_format="text"):
"""Creates a report summarizing the top contributing features for the best and worst points in the dataset as measured by error to true labels.
XGBoost models and CatBoost multiclass classifiers are not currently supported.
Arguments:
pipeline (PipelineBase): Fitted pipeline whose predictions we want to explain with SHAP.
input_features (ww.DataTable, pd.DataFrame): Input data to evaluate the pipeline on.
y_true (ww.DataColumn, pd.Series): True labels for the input data.
num_to_explain (int): How many of the best, worst, random data points to explain.
top_k_features (int): How many of the highest/lowest contributing feature to include in the table for each
data point.
include_shap_values (bool): Whether SHAP values should be included in the table. Default is False.
metric (callable): The metric used to identify the best and worst points in the dataset. Function must accept
the true labels and predicted value or probabilities as the only arguments and lower values
must be better. By default, this will be the absolute error for regression problems and cross entropy loss
for classification problems.
output_format (str): Either "text" or "dict". Default is "text".
Returns:
str, dict, or pd.DataFrame - A report explaining the top contributing features for the best/worst predictions in the input_features.
For each of the best/worst rows of input_features, the predicted values, true labels, metric value,
feature names, prediction contribution, and SHAP Value (optional) will be listed.
Raises:
ValueError: if input_features does not have more than twice the requested features to explain.
ValueError: if y_true and input_features have mismatched lengths.
ValueError: if an output_format outside of "text", "dict" or "dataframe is provided.
"""
input_features = infer_feature_types(input_features)
input_features = _convert_woodwork_types_wrapper(
input_features.to_dataframe())
y_true = infer_feature_types(y_true)
y_true = _convert_woodwork_types_wrapper(y_true.to_series())
if not (input_features.shape[0] >= num_to_explain * 2):
raise ValueError(
f"Input features must be a dataframe with more than {num_to_explain * 2} rows! "
"Convert to a dataframe and select a smaller value for num_to_explain if you do not have "
"enough data.")
if y_true.shape[0] != input_features.shape[0]:
raise ValueError(
"Parameters y_true and input_features must have the same number of data points. Received: "
f"true labels: {y_true.shape[0]} and {input_features.shape[0]}")
if output_format not in {"text", "dict", "dataframe"}:
raise ValueError(
f"Parameter output_format must be either text, dict, or dataframe. Received {output_format}"
)
if not metric:
metric = DEFAULT_METRICS[pipeline.problem_type]
try:
if is_regression(pipeline.problem_type):
if is_time_series(pipeline.problem_type):
y_pred = pipeline.predict(input_features, y=y_true).to_series()
else:
y_pred = pipeline.predict(input_features).to_series()
y_pred_values = None
y_true_no_nan, y_pred_no_nan = drop_rows_with_nans(y_true, y_pred)
errors = metric(y_true_no_nan, y_pred_no_nan)
else:
if is_time_series(pipeline.problem_type):
y_pred = pipeline.predict_proba(input_features,
y=y_true).to_dataframe()
y_pred_values = pipeline.predict(input_features,
y=y_true).to_series()
else:
y_pred = pipeline.predict_proba(input_features).to_dataframe()
y_pred_values = pipeline.predict(input_features).to_series()
y_true_no_nan, y_pred_no_nan, y_pred_values_no_nan = drop_rows_with_nans(
y_true, y_pred, y_pred_values)
errors = metric(pipeline._encode_targets(y_true_no_nan),
y_pred_no_nan)
except Exception as e:
tb = traceback.format_tb(sys.exc_info()[2])
raise PipelineScoreError(exceptions={metric.__name__: (e, tb)},
scored_successfully={})
errors = pd.Series(errors, index=y_pred_no_nan.index)
sorted_scores = errors.sort_values()
best_indices = sorted_scores.index[:num_to_explain]
worst_indices = sorted_scores.index[-num_to_explain:]
index_list = best_indices.tolist() + worst_indices.tolist()
pipeline_features = pipeline.compute_estimator_features(
input_features, y_true).to_dataframe()
data = _ReportData(pipeline, pipeline_features, input_features, y_true,
y_pred, y_pred_values, errors, index_list, metric)
report_creator = _report_creator_factory(
data,
report_type="explain_predictions_best_worst",
output_format=output_format,
top_k_features=top_k_features,
include_shap_values=include_shap_values,
num_to_explain=num_to_explain)
return report_creator(data)
def test_drop_nan(data, expected):
no_nan_1, no_nan_2 = drop_rows_with_nans(*data)
_check_equality(no_nan_1, expected[0], check_index_type=False)
_check_equality(no_nan_2, expected[1], check_index_type=False)