def compute_permutation_feature_importance( X: pd.DataFrame, y: pd.Series, predict_func: Callable[..., np.ndarray], eval_metric: Scorer, features: list = None, subsample_size=None, num_shuffle_sets: int = None, predict_func_kwargs: dict = None, transform_func: Callable[..., pd.DataFrame] = None, transform_func_kwargs: dict = None, time_limit: float = None, silent=False, log_prefix='', importance_as_list=False) -> pd.DataFrame: """ Computes a trained model's feature importance via permutation shuffling (https://explained.ai/rf-importance/). A feature's importance score represents the performance drop that results when the model makes predictions on a perturbed copy of the dataset where this feature's values have been randomly shuffled across rows. A feature score of 0.01 would indicate that the predictive performance dropped by 0.01 when the feature was randomly shuffled. The higher the score a feature has, the more important it is to the model's performance. If a feature has a negative score, this means that the feature is likely harmful to the final model, and a model trained with the feature removed would be expected to achieve a better predictive performance. Note that calculating feature importance can be a very computationally expensive process, particularly if the model uses hundreds or thousands of features. In many cases, this can take longer than the original model training. Note: For highly accurate stddev and z_score estimates, it is recommend to set `subsample_size` to at least 5,000 if possible and `num_shuffle_sets` to at least 10. Parameters ---------- X : pd.DataFrame Validation data to permute when calculating feature importances. Do not use training data as it will result in overfit feature importances. y : pd.Series Label values of X. The index of X and y must align. predict_func : Callable[..., np.ndarray] Function that computes model predictions or prediction probabilities on input data. Output must be in the form of a numpy ndarray or pandas Series or DataFrame. Output `y_pred` must be in a form acceptable as input to `eval_metric(y, y_pred)`. If using a fit model object, this is typically `model.predict` or `model.predict_proba`, depending on the `eval_metric` being used. If `eval_metric.needs_pred==True`, use `model.predict`, otherwise use `model.predict_proba`. eval_metric : Scorer Object that computes a score given ground truth labels and predictions or prediction probabilities (depending on the type of metric). If using a fit model object, this is typically `model.eval_metric`. Feature importances will be based on the delta permutation shuffling has on the score produced by `eval_metric`. features : list, default None List of features to calculate importances for. If None, all features' importances will be calculated. subsample_size : int, default None The amount of data rows to sample when computing importances. Higher values will improve the quality of feature importance estimates, but linearly increase the runtime. If None, all provided data will be used. num_shuffle_sets : int, default None The number of different permutation shuffles of the data that are evaluated. Shuffle sets are generated with different random seeds and importances are averaged across all shuffle sets to get the final feature importance values. Higher values will improve the quality of feature importance estimates, but linearly increase the runtime. `subsample_size` should be increased before `num_shuffle_sets` if runtime is a concern. Defaults to 1 if `time_limit` is None or 10 if `time_limit` is specified. When `num_shuffle_sets` is greater than 1, feature importance standard deviation and z-score will additionally be computed by using the results of each shuffle set as samples. predict_func_kwargs : dict, default {} Keyword arguments to be appended to calls to `predict_func(X, **kwargs)`. transform_func : Callable[..., pd.DataFrame], default None Transformation function that takes the raw input and transforms it row-wise to the input expected by `predict_func`. Common examples include `model.preprocess` and `feature_generator.transform`. If None, then no transformation is done on the data prior to calling `predict_func`. This is necessary to compute importance of original data features in `X` prior to their transformation assuming `predict_func` does not perform the transformation already. Example: `transform_func` is necessary to compute the importance of a text feature prior to being transformed into ngrams by `transform_func` when `predict_func` expects ngram features as input. transform_func_kwargs : dict, default {} Keyword arguments to be appended to calls to `transform_func(X, **kwargs)`. time_limit : float, default None Time in seconds to limit the calculation of feature importance. If None, feature importance will calculate without early stopping. A minimum of 1 full shuffle set will always be evaluated. If a shuffle set evaluation takes longer than `time_limit`, the method will take the length of a shuffle set evaluation to return regardless of the `time_limit`. If `num_shuffle_sets==1`, `time_limit` will be ignored. silent : bool, default False Whether to suppress logging output. log_prefix : str, default '' Prefix to add to logging statements. importance_as_list : bool, default False Whether to return the 'importance' column values as a list of the importance from each shuffle (True) or a single averaged value (False). Returns ------- Pandas `pandas.DataFrame` of feature importance scores with 4 columns: index: The feature name. 'importance': The estimated feature importance score. 'stddev': The standard deviation of the feature importance score. If NaN, then not enough num_shuffle_sets were used to calculate a variance. 'p_value': P-value for a statistical t-test of the null hypothesis: importance = 0, vs the (one-sided) alternative: importance > 0. Features with low p-value appear confidently useful to the predictor, while the other features may be useless to the predictor (or even harmful to include in its training data). A p-value of 0.01 indicates that there is a 1% chance that the feature is useless or harmful, and a 99% chance that the feature is useful. A p-value of 0.99 indicates that there is a 99% chance that the feature is useless or harmful, and a 1% chance that the feature is useful. 'n': The number of shuffles performed to estimate importance score (corresponds to sample-size used to determine confidence interval for true score). """ if num_shuffle_sets is None: num_shuffle_sets = 1 if time_limit is None else 10 time_start = time.time() if predict_func_kwargs is None: predict_func_kwargs = dict() if transform_func_kwargs is None: transform_func_kwargs = dict() if features is None: features = list(X.columns) num_features = len(features) if subsample_size is not None: num_rows = min(len(X), subsample_size) else: num_rows = len(X) subsample = num_rows < len(X) if not silent: logging_message = f'{log_prefix}Computing feature importance via permutation shuffling for {num_features} features using {num_rows} rows with {num_shuffle_sets} shuffle sets...' if time_limit is not None: logging_message = f'{logging_message} Time limit: {time_limit}s...' logger.log(20, logging_message) time_permutation_start = time.time() fi_dict_list = [] shuffle_repeats_completed = 0 log_final_suffix = '' X_orig = X y_orig = y feature_batch_count = None X_raw = None score_baseline = None # TODO: Can speedup shuffle_repeats by incorporating into X_raw (do multiple repeats in a single predict call) for shuffle_repeat in range(num_shuffle_sets): fi = dict() if subsample: # TODO: Stratify? We currently don't know in this function the problem_type (could pass as additional arg). X = X_orig.sample(subsample_size, random_state=shuffle_repeat) y = y_orig.loc[X.index] if subsample or shuffle_repeat == 0: time_start_score = time.time() X_transformed = X if transform_func is None else transform_func( X, **transform_func_kwargs) y_pred = predict_func(X_transformed, **predict_func_kwargs) score_baseline = eval_metric(y, y_pred) if shuffle_repeat == 0: if not silent: time_score = time.time() - time_start_score time_estimated = ( (num_features + 1) * time_score ) * num_shuffle_sets + time_start_score - time_start time_estimated_per_set = time_estimated / num_shuffle_sets logger.log( 20, f'{log_prefix}\t{round(time_estimated, 2)}s\t= Expected runtime ({round(time_estimated_per_set, 2)}s per shuffle set)' ) if transform_func is None: feature_batch_count = _get_safe_fi_batch_count( X=X, num_features=num_features) else: feature_batch_count = _get_safe_fi_batch_count( X=X, num_features=num_features, X_transformed=X_transformed) # creating copy of original data N=feature_batch_count times for parallel processing X_raw = pd.concat([X.copy() for _ in range(feature_batch_count)], ignore_index=True, sort=False).reset_index(drop=True) row_count = len(X) X_shuffled = shuffle_df_rows(X=X, seed=shuffle_repeat) for i in range(0, num_features, feature_batch_count): parallel_computed_features = features[i:i + feature_batch_count] # if final iteration, leaving only necessary part of X_raw num_features_processing = len(parallel_computed_features) final_iteration = i + num_features_processing == num_features row_index = 0 for feature in parallel_computed_features: row_index_end = row_index + row_count X_raw.loc[row_index:row_index_end - 1, feature] = X_shuffled[feature].values row_index = row_index_end if (num_features_processing < feature_batch_count) and final_iteration: X_raw_transformed = X_raw.loc[:row_count * num_features_processing - 1] X_raw_transformed = X_raw_transformed if transform_func is None else transform_func( X_raw_transformed, **transform_func_kwargs) else: X_raw_transformed = X_raw if transform_func is None else transform_func( X_raw, **transform_func_kwargs) y_pred = predict_func(X_raw_transformed, **predict_func_kwargs) row_index = 0 for feature in parallel_computed_features: # calculating importance score for given feature row_index_end = row_index + row_count y_pred_cur = y_pred[row_index:row_index_end] score = eval_metric(y, y_pred_cur) fi[feature] = score_baseline - score # resetting to original values for processed feature X_raw.loc[row_index:row_index_end - 1, feature] = X[feature].values row_index = row_index_end fi_dict_list.append(fi) shuffle_repeats_completed = shuffle_repeat + 1 if time_limit is not None and shuffle_repeat != (num_shuffle_sets - 1): time_now = time.time() time_left = time_limit - (time_now - time_start) time_permutation_average = (time_now - time_permutation_start) / ( shuffle_repeat + 1) if time_left < (time_permutation_average * 1.1): log_final_suffix = ' (Early stopping due to lack of time...)' break fi_list_dict = dict() for val in fi_dict_list: for key in val: if key not in fi_list_dict: fi_list_dict[key] = [] fi_list_dict[key].append(val[key]) fi_df = _compute_fi_with_stddev(fi_list_dict, importance_as_list=importance_as_list) if not silent: logger.log( 20, f'{log_prefix}\t{round(time.time() - time_start, 2)}s\t= Actual runtime (Completed {shuffle_repeats_completed} of {num_shuffle_sets} shuffle sets){log_final_suffix}' ) return fi_df
def compute_permutation_importance(self, X, y, features: list, preprocess=True, silent=False) -> dict: time_start = time.time() feature_count = len(features) if not silent: logger.log(20, f'Computing permutation importance for {feature_count} features on {self.name} ...') if preprocess: X = self.preprocess(X) time_start_score = time.time() model_score_base = self.score(X=X, y=y, preprocess=False) time_score = time.time() - time_start_score if not silent: time_estimated = (feature_count + 1) * time_score + time_start_score - time_start logger.log(20, f'\t{round(time_estimated, 2)}s\t= Expected runtime') X_shuffled = shuffle_df_rows(X=X, seed=0) row_count = X.shape[0] # calculating maximum number of features, which is safe to process parallel X_memory_ratio_max = 0.2 compute_count_max = 200 X_size_bytes = sys.getsizeof(pickle.dumps(X, protocol=4)) available_mem = psutil.virtual_memory().available X_memory_ratio = X_size_bytes / available_mem compute_count_safe = math.floor(X_memory_ratio_max / X_memory_ratio) compute_count = max(1, min(compute_count_max, compute_count_safe)) compute_count = min(compute_count, feature_count) # creating copy of original data N=compute_count times for parallel processing X_raw = pd.concat([X.copy() for _ in range(compute_count)], ignore_index=True, sort=False).reset_index(drop=True) # TODO: Make this faster by multi-threading? permutation_importance_dict = {} for i in range(0, feature_count, compute_count): parallel_computed_features = features[i:i + compute_count] # if final iteration, leaving only necessary part of X_raw num_features_processing = len(parallel_computed_features) final_iteration = i + num_features_processing == feature_count if (num_features_processing < compute_count) and final_iteration: X_raw = X_raw.loc[:row_count * num_features_processing - 1] row_index = 0 for feature in parallel_computed_features: row_index_end = row_index + row_count X_raw.loc[row_index:row_index_end - 1, feature] = X_shuffled[feature].values row_index = row_index_end if self.metric_needs_y_pred: y_pred = self.predict(X_raw, preprocess=False) else: y_pred = self.predict_proba(X_raw, preprocess=False) row_index = 0 for feature in parallel_computed_features: # calculating importance score for given feature row_index_end = row_index + row_count y_pred_cur = y_pred[row_index:row_index_end] score = self.eval_metric(y, y_pred_cur) permutation_importance_dict[feature] = model_score_base - score if not final_iteration: # resetting to original values for processed feature X_raw.loc[row_index:row_index_end - 1, feature] = X[feature].values row_index = row_index_end if not silent: logger.log(20, f'\t{round(time.time() - time_start, 2)}s\t= Actual runtime') return permutation_importance_dict