def __init__(self, model, dataset, target_column, categorical_features=None): """Defines the ErrorAnalysisManager for discovering errors in a model. :param model: The model to analyze errors on. A model that implements sklearn.predict or sklearn.predict_proba or function that accepts a 2d ndarray. :type model: object :param dataset: The dataset including the label column. :type dataset: pandas.DataFrame :param target_column: The name of the label column. :type target_column: str """ self._true_y = dataset[target_column] self._dataset = dataset.drop(columns=[target_column]) self._feature_names = list(self._dataset.columns) self._categorical_features = categorical_features self._ea_config_list = [] self._ea_report_list = [] self._analyzer = ModelAnalyzer(model, self._dataset, self._true_y, self._feature_names, self._categorical_features)
def run_error_analyzer(model, X_test, y_test, feature_names, categorical_features, metric=Metrics.ERROR_RATE): model_analyzer = ModelAnalyzer(model, X_test, y_test, feature_names, categorical_features, metric=metric) root_stats = model_analyzer.compute_root_stats() metric_name = metric_to_display_name[metric] total = len(X_test) assert root_stats[RootKeys.METRIC_NAME] == metric_name assert root_stats[RootKeys.TOTAL_SIZE] == total assert root_stats[RootKeys.ERROR_COVERAGE] == 100 if metric == Metrics.ERROR_RATE: diff = model_analyzer.get_diff() error = sum(diff) metric_value = (error / total) * 100 else: metric_func = metric_to_func[metric] metric_value = metric_func(model_analyzer.pred_y, model_analyzer.true_y) assert root_stats[RootKeys.METRIC_VALUE] == metric_value
def __init__(self, analysis: ModelAnalysis): """Initialize the Explanation Dashboard Input. :param analysis: An ModelAnalysis object that represents an explanation. :type analysis: ModelAnalysis """ self._analysis = analysis model = analysis.model self._is_classifier = model is not None\ and hasattr(model, SKLearn.PREDICT_PROBA) and \ model.predict_proba is not None self._dataframeColumns = None self.dashboard_input = ModelAnalysisDashboardData() self.dashboard_input.dataset = self._get_dataset() self._feature_length = len(self.dashboard_input.dataset.featureNames) self._row_length = len(self.dashboard_input.dataset.features) self.dashboard_input.modelExplanationData = [ self._get_interpret(i) for i in self._analysis.explainer.get() ] self.dashboard_input.errorAnalysisConfig = [ self._get_error_analysis(i) for i in self._analysis.error_analysis.list()["reports"] ] x_test = analysis.test.drop(columns=[analysis.target_column]) y_test = analysis.test[analysis.target_column] self._error_analyzer = ModelAnalyzer(model, x_test, y_test, x_test.columns.values.tolist(), analysis.categorical_features)
def run_error_analyzer(model, X_test, y_test, feature_names, analyzer_type, categorical_features=None, tree_features=None, max_depth=3, num_leaves=31, min_child_samples=20, filters=None, composite_filters=None, metric=None, model_task=None): if analyzer_type == AnalyzerType.MODEL: error_analyzer = ModelAnalyzer(model, X_test, y_test, feature_names, categorical_features, metric=metric, model_task=model_task) else: pred_y = model.predict(X_test) error_analyzer = PredictionsAnalyzer(pred_y, X_test, y_test, feature_names, categorical_features, metric=metric, model_task=model_task) if tree_features is None: tree_features = feature_names tree = error_analyzer.compute_error_tree( tree_features, filters, composite_filters, max_depth=max_depth, num_leaves=num_leaves, min_child_samples=min_child_samples) validation_data = X_test if filters is not None or composite_filters is not None: validation_data = filter_from_cohort(error_analyzer, filters, composite_filters) y_test = validation_data[TRUE_Y] validation_data = validation_data.drop(columns=[TRUE_Y, ROW_INDEX]) if not isinstance(X_test, pd.DataFrame): validation_data = validation_data.values validation_data_len = len(validation_data) assert tree is not None assert len(tree) > 0 assert ERROR in tree[0] assert ID in tree[0] assert PARENTID in tree[0] assert tree[0][PARENTID] is None assert SIZE in tree[0] assert tree[0][SIZE] == validation_data_len for node in tree: assert node[SIZE] >= min(min_child_samples, validation_data_len)
def run_error_analyzer(model, x_test, y_test, feature_names, categorical_features): model_analyzer = ModelAnalyzer(model, x_test, y_test, feature_names, categorical_features) scores = model_analyzer.compute_importances() diff = model.predict(model_analyzer.dataset) != model_analyzer.true_y assert isinstance(scores, list) assert len(scores) == len(feature_names) # If model predicted perfectly, assert all scores are zeros if not any(diff): assert all(abs(score - 0) < TOL for score in scores) else: assert any(score != 0 for score in scores)
def run_error_analyzer(model, x_test, y_test, feature_names, categorical_features): error_analyzer = ModelAnalyzer(model, x_test, y_test, feature_names, categorical_features) # features, filters, composite_filters features = [feature_names[0], feature_names[1]] filters = None composite_filters = None json_matrix = error_analyzer.compute_matrix(features, filters, composite_filters) expected_count = len(x_test) expected_false_count = sum(model.predict(x_test) != y_test) validate_matrix(json_matrix, expected_count, expected_false_count)
def test_large_data_surrogate_error_tree(self): # validate tree trains quickly for large data X_train, y_train, X_test, y_test, _ = \ create_binary_classification_dataset(100) feature_names = list(X_train.columns) model = create_sklearn_random_forest_regressor(X_train, y_train) X_test, y_test = replicate_dataset(X_test, y_test) assert X_test.shape[0] > 1000000 t0 = time.time() categorical_features = [] model_analyzer = ModelAnalyzer(model, X_test, y_test, feature_names, categorical_features) max_depth = 3 num_leaves = 31 min_child_samples = 20 categories_reindexed = [] cat_ind_reindexed = [] diff = model_analyzer.get_diff() surrogate = create_surrogate_model(model_analyzer, X_test, diff, max_depth, num_leaves, min_child_samples, cat_ind_reindexed) t1 = time.time() execution_time = t1 - t0 print( "creating surrogate model took {} seconds".format(execution_time)) # assert we don't take too long to train the tree on 1 million rows # note we train on >1 million rows in ~1 second assert execution_time < 20 model_json = surrogate._Booster.dump_model() tree_structure = model_json["tree_info"][0]['tree_structure'] max_split_index = get_max_split_index(tree_structure) + 1 assert max_split_index == 3 cache_subtree_features(tree_structure, feature_names) pred_y = model_analyzer.model.predict(X_test) traversed_X_test = X_test.copy() traversed_X_test[DIFF] = diff traversed_X_test[TRUE_Y] = y_test traversed_X_test[PRED_Y] = pred_y t2 = time.time() tree = traverse(traversed_X_test, tree_structure, max_split_index, (categories_reindexed, cat_ind_reindexed), [], feature_names, metric=model_analyzer.metric, classes=model_analyzer.classes) t3 = time.time() execution_time = t3 - t2 print("traversing tree took {} seconds".format(execution_time)) assert tree is not None
def run_error_analyzer(model, X_test, y_test, feature_names, categorical_features, model_task, filters=None, composite_filters=None, matrix_features=None): error_analyzer = ModelAnalyzer(model, X_test, y_test, feature_names, categorical_features, model_task=model_task) # features, filters, composite_filters if matrix_features is None: features = [feature_names[0], feature_names[1]] else: features = matrix_features json_matrix = error_analyzer.compute_matrix(features, filters, composite_filters) validation_data = X_test if filters is not None or composite_filters is not None: validation_data = filter_from_cohort(X_test, filters, composite_filters, feature_names, y_test, categorical_features, error_analyzer.categories) y_test = validation_data[TRUE_Y] validation_data = validation_data.drop(columns=[TRUE_Y, ROW_INDEX]) if not isinstance(X_test, pd.DataFrame): validation_data = validation_data.values expected_count = len(validation_data) metric = error_analyzer.metric if metric == Metrics.ERROR_RATE: expected_error = sum(model.predict(validation_data) != y_test) elif metric == Metrics.MEAN_SQUARED_ERROR: func = metric_to_func[metric] pred_y = model.predict(validation_data) expected_error = func(y_test, pred_y) else: raise NotImplementedError( "Metric {} validation not supported yet".format(metric)) validate_matrix(json_matrix, expected_count, expected_error, features, metric=metric)
def compute(self): """Creates an ErrorReport by running the error analyzer on the model. """ for config in self._ea_config_list: if config.is_computed: continue config.is_computed = True analyzer = ModelAnalyzer(self._model, self._train, self._y_train, self._feature_names, self._categorical_features) max_depth = config.max_depth num_leaves = config.num_leaves report = analyzer.create_error_report(config.filter_features, max_depth=max_depth, num_leaves=num_leaves) self._ea_report_list.append(report)
def _load(path, model_analysis): """Load the ErrorAnalysisManager from the given path. :param path: The directory path to load the ErrorAnalysisManager from. :type path: str :param model_analysis: The loaded parent ModelAnalysis. :type model_analysis: ModelAnalysis """ # create the ErrorAnalysisManager without any properties using # the __new__ function, similar to pickle inst = ErrorAnalysisManager.__new__(ErrorAnalysisManager) top_dir = Path(path) reports_path = top_dir / REPORTS with open(reports_path, 'r') as file: ea_report_list = json.load(file, object_hook=as_error_report) inst.__dict__['_ea_report_list'] = ea_report_list config_path = top_dir / CONFIG with open(config_path, 'r') as file: ea_config_list = json.load(file, object_hook=as_error_config) inst.__dict__['_ea_config_list'] = ea_config_list categorical_features = model_analysis.categorical_features inst.__dict__['_categorical_features'] = categorical_features target_column = model_analysis.target_column true_y = model_analysis.test[target_column] dataset = model_analysis.test.drop(columns=[target_column]) inst.__dict__['_dataset'] = dataset inst.__dict__['_true_y'] = true_y feature_names = list(dataset.columns) inst.__dict__['_feature_names'] = feature_names inst.__dict__['_analyzer'] = ModelAnalyzer(model_analysis.model, dataset, true_y, feature_names, categorical_features) return inst
def run_error_analyzer(validation_data, model, X_test, y_test, feature_names, categorical_features, model_task, filters=None, composite_filters=None, is_empty_validation_data=False): error_analyzer = ModelAnalyzer(model, X_test, y_test, feature_names, categorical_features, model_task=model_task) filtered_data = filter_from_cohort(error_analyzer, filters, composite_filters) # validate there is some data selected for each of the filters if is_empty_validation_data: assert validation_data.shape[0] == 0 else: assert validation_data.shape[0] > 0 assert validation_data.equals(filtered_data)
def run_error_analyzer(model, X_test, y_test, feature_names, categorical_features, model_task, filters=None, composite_filters=None, matrix_features=None, quantile_binning=False, num_bins=BIN_THRESHOLD, metric=None): error_analyzer = ModelAnalyzer(model, X_test, y_test, feature_names, categorical_features, model_task=model_task, metric=metric) # features, filters, composite_filters if matrix_features is None: features = [feature_names[0], feature_names[1]] else: features = matrix_features matrix = error_analyzer.compute_matrix(features, filters, composite_filters, quantile_binning=quantile_binning, num_bins=num_bins) validation_data = X_test if filters is not None or composite_filters is not None: validation_data = filter_from_cohort(error_analyzer, filters, composite_filters) y_test = validation_data[TRUE_Y] validation_data = validation_data.drop(columns=[TRUE_Y, ROW_INDEX]) if not isinstance(X_test, pd.DataFrame): validation_data = validation_data.values expected_count = len(validation_data) metric = error_analyzer.metric expected_error = get_expected_metric_error(error_analyzer, metric, model, validation_data, y_test) validate_matrix(matrix, expected_count, expected_error, features, metric=metric)
def run_error_analyzer(model, x_test, y_test, feature_names, categorical_features): error_analyzer = ModelAnalyzer(model, x_test, y_test, feature_names, categorical_features) # features, filters, composite_filters features = [feature_names[0], feature_names[1]] filters = None composite_filters = None json_tree = error_analyzer.compute_error_tree(features, filters, composite_filters) assert json_tree is not None assert len(json_tree) > 0 assert ERROR in json_tree[0] assert ID in json_tree[0] assert PARENTID in json_tree[0] assert json_tree[0][PARENTID] is None assert SIZE in json_tree[0] assert json_tree[0][SIZE] == len(x_test)
def run_error_analyzer(model, X_test, y_test, feature_names, categorical_features, tree_features=None): error_analyzer = ModelAnalyzer(model, X_test, y_test, feature_names, categorical_features) if tree_features is None: tree_features = feature_names filters = None composite_filters = None json_tree = error_analyzer.compute_error_tree(tree_features, filters, composite_filters) assert json_tree is not None assert len(json_tree) > 0 assert ERROR in json_tree[0] assert ID in json_tree[0] assert PARENTID in json_tree[0] assert json_tree[0][PARENTID] is None assert SIZE in json_tree[0] assert json_tree[0][SIZE] == len(X_test)
def test_large_data_importances(self): # mutual information can be very costly for large number of rows # hence, assert we downsample to compute importances for large data X_train, y_train, X_test, y_test, _ = \ create_binary_classification_dataset(100) feature_names = list(X_train.columns) model = create_sklearn_random_forest_regressor(X_train, y_train) X_test, y_test = replicate_dataset(X_test, y_test) assert X_test.shape[0] > 1000000 t0 = time.time() categorical_features = [] model_analyzer = ModelAnalyzer(model, X_test, y_test, feature_names, categorical_features) model_analyzer.compute_importances() t1 = time.time() execution_time = t1 - t0 print(execution_time) # assert we don't take too long and downsample the dataset # note execution time is in seconds assert execution_time < 20
def run_error_analyzer(model, X_test, y_test, feature_names, categorical_features, expect_user_warnings=False): if expect_user_warnings and pd.__version__[0] == '0': with pytest.warns(UserWarning, match='which has issues with pandas version'): model_analyzer = ModelAnalyzer(model, X_test, y_test, feature_names, categorical_features) else: model_analyzer = ModelAnalyzer(model, X_test, y_test, feature_names, categorical_features) error_report1 = model_analyzer.create_error_report(filter_features=None, max_depth=3, num_leaves=None) error_report2 = model_analyzer.create_error_report() assert error_report1.id != error_report2.id # validate uuids in correct format assert is_valid_uuid(error_report1.id) assert is_valid_uuid(error_report2.id) json_str1 = error_report1.to_json() json_str2 = error_report2.to_json() assert json_str1 != json_str2 error_report_deserialized = ErrorReport.from_json(json_str1) assert error_report_deserialized.id == error_report1.id assert error_report_deserialized.json_matrix == error_report1.json_matrix assert error_report_deserialized.json_tree == error_report1.json_tree
def _load(path, rai_insights): """Load the ErrorAnalysisManager from the given path. :param path: The directory path to load the ErrorAnalysisManager from. :type path: str :param rai_insights: The loaded parent RAIInsights. :type rai_insights: RAIInsights :return: The ErrorAnalysisManager manager after loading. :rtype: ErrorAnalysisManager """ # create the ErrorAnalysisManager without any properties using # the __new__ function, similar to pickle inst = ErrorAnalysisManager.__new__(ErrorAnalysisManager) ea_config_list = [] ea_report_list = [] all_ea_dirs = DirectoryManager.list_sub_directories(path) for ea_dir in all_ea_dirs: directory_manager = DirectoryManager(parent_directory_path=path, sub_directory_name=ea_dir) config_path = (directory_manager.get_config_directory() / 'config.json') with open(config_path, 'r') as file: ea_config = json.load(file, object_hook=as_error_config) ea_config_list.append(ea_config) report_path = (directory_manager.get_data_directory() / 'report.json') with open(report_path, 'r') as file: ea_report = json.load(file, object_hook=as_error_report) # Validate the serialized output against schema schema = ErrorAnalysisManager._get_error_analysis_schema() jsonschema.validate(json.loads(ea_report.to_json()), schema) ea_report_list.append(ea_report) inst.__dict__['_ea_report_list'] = ea_report_list inst.__dict__['_ea_config_list'] = ea_config_list categorical_features = rai_insights.categorical_features inst.__dict__['_categorical_features'] = categorical_features target_column = rai_insights.target_column true_y = rai_insights.test[target_column] dataset = rai_insights.test.drop(columns=[target_column]) inst.__dict__['_dataset'] = dataset inst.__dict__['_true_y'] = true_y feature_names = list(dataset.columns) inst.__dict__['_feature_names'] = feature_names inst.__dict__['_analyzer'] = ModelAnalyzer(rai_insights.model, dataset, true_y, feature_names, categorical_features) return inst
def __init__(self, model: Any, dataset: pd.DataFrame, target_column: str, classes: Optional[List] = None, categorical_features: Optional[List[str]] = None): """Creates an ErrorAnalysisManager object. :param model: The model to analyze errors on. A model that implements sklearn.predict or sklearn.predict_proba or function that accepts a 2d ndarray. :type model: object :param dataset: The dataset including the label column. :type dataset: pandas.DataFrame :param target_column: The name of the label column. :type target_column: str :param classes: Class names as a list of strings. The order of the class names should match that of the model output. Only required if analyzing a classifier. :type classes: list :param categorical_features: The categorical feature names. :type categorical_features: list[str] """ self._true_y = dataset[target_column] self._dataset = dataset.drop(columns=[target_column]) self._feature_names = list(self._dataset.columns) self._classes = classes self._categorical_features = categorical_features self._ea_config_list = [] self._ea_report_list = [] self._analyzer = ModelAnalyzer(model, self._dataset, self._true_y, self._feature_names, self._categorical_features, classes=self._classes)
def setup_pyspark(self, model, dataset, true_y, classes, features, categorical_features, true_y_dataset, pred_y, pred_y_dataset, model_task, metric, max_depth, num_leaves, min_child_samples, sample_dataset, model_available): self._error_analyzer = ModelAnalyzer(model, dataset, true_y, features, categorical_features, model_task, metric, classes) sample = dataset.to_spark().limit(100) scored_sample = model.transform(sample) pd_sample = scored_sample.toPandas() predicted_y = pd_sample["prediction"] predicted_y = self.predicted_y_to_list(predicted_y) true_y = pd_sample[true_y] pd_sample = pd_sample[features] list_dataset = convert_to_list(pd_sample) self.setup_visualization_input(classes, predicted_y, list_dataset, true_y, features)
def run_error_analyzer(model, X_test, y_test, feature_names, categorical_features, expect_user_warnings=False, filter_features=None): if expect_user_warnings and pd.__version__[0] == '0': with pytest.warns(UserWarning, match='which has issues with pandas version'): model_analyzer = ModelAnalyzer(model, X_test, y_test, feature_names, categorical_features) else: model_analyzer = ModelAnalyzer(model, X_test, y_test, feature_names, categorical_features) report1 = model_analyzer.create_error_report(filter_features, max_depth=3, num_leaves=None, compute_importances=True) report2 = model_analyzer.create_error_report() assert report1.id != report2.id # validate uuids in correct format assert is_valid_uuid(report1.id) assert is_valid_uuid(report2.id) json_str1 = report1.to_json() json_str2 = report2.to_json() assert json_str1 != json_str2 # validate deserialized error report json ea_deserialized = ErrorReport.from_json(json_str1) assert ea_deserialized.id == report1.id assert ea_deserialized.matrix == report1.matrix assert ea_deserialized.tree == report1.tree assert ea_deserialized.tree_features == report1.tree_features assert ea_deserialized.matrix_features == report1.matrix_features assert ea_deserialized.importances == report1.importances assert ea_deserialized.root_stats == report1.root_stats if not filter_features: assert ea_deserialized.matrix is None else: assert ea_deserialized.matrix is not None # validate error report does not modify original dataset in ModelAnalyzer if isinstance(X_test, pd.DataFrame): assert X_test.equals(model_analyzer.dataset) else: assert np.array_equal(X_test, model_analyzer.dataset)
def test_traverse_tree(self): X_train, X_test, y_train, y_test, categorical_features = \ create_adult_census_data() model = create_kneighbors_classifier(X_train, y_train) feature_names = list(X_train.columns) error_analyzer = ModelAnalyzer(model, X_test, y_test, feature_names, categorical_features) categorical_info = get_categorical_info(error_analyzer, feature_names) cat_ind_reindexed, categories_reindexed = categorical_info pred_y = model.predict(X_test) diff = pred_y != y_test max_depth = 3 num_leaves = 31 surrogate = create_surrogate_model(error_analyzer, X_test, diff, max_depth, num_leaves, cat_ind_reindexed) model_json = surrogate._Booster.dump_model() tree_structure = model_json["tree_info"][0]['tree_structure'] max_split_index = get_max_split_index(tree_structure) + 1 filtered_indexed_df = X_test filtered_indexed_df[DIFF] = diff filtered_indexed_df[TRUE_Y] = y_test filtered_indexed_df[PRED_Y] = pred_y json_tree = traverse(filtered_indexed_df, tree_structure, max_split_index, (categories_reindexed, cat_ind_reindexed), [], feature_names, metric=error_analyzer.metric) # create dictionary from json tree id to values json_tree_dict = {} for entry in json_tree: json_tree_dict[entry['id']] = entry validate_traversed_tree(tree_structure, json_tree_dict, max_split_index, feature_names)
class ErrorAnalysisManager(BaseManager): """Defines the ErrorAnalysisManager for discovering errors in a model. :param model: The model to analyze errors on. A model that implements sklearn.predict or sklearn.predict_proba or function that accepts a 2d ndarray. :type model: object :param dataset: The dataset including the label column. :type dataset: pandas.DataFrame :param target_column: The name of the label column. :type target_column: str """ def __init__(self, model, dataset, target_column, categorical_features=None): """Defines the ErrorAnalysisManager for discovering errors in a model. :param model: The model to analyze errors on. A model that implements sklearn.predict or sklearn.predict_proba or function that accepts a 2d ndarray. :type model: object :param dataset: The dataset including the label column. :type dataset: pandas.DataFrame :param target_column: The name of the label column. :type target_column: str """ self._true_y = dataset[target_column] self._dataset = dataset.drop(columns=[target_column]) self._feature_names = list(self._dataset.columns) self._categorical_features = categorical_features self._ea_config_list = [] self._ea_report_list = [] self._analyzer = ModelAnalyzer(model, self._dataset, self._true_y, self._feature_names, self._categorical_features) def add(self, max_depth=3, num_leaves=31, filter_features=None): """Add an error analyzer to be computed later. :param max_depth: The maximum depth of the tree. :type max_depth: int :param num_leaves: The number of leaves in the tree. :type num_leaves: int :param filter_features: One or two features to use for the matrix filter. :type filter_features: list """ ea_config = ErrorAnalysisConfig(max_depth=max_depth, num_leaves=num_leaves, filter_features=filter_features) is_duplicate = ea_config.is_duplicate(self._ea_config_list) if is_duplicate: raise DuplicateManagerConfigException( "Duplicate config specified for error analysis," "config already added") else: self._ea_config_list.append(ea_config) def compute(self): """Creates an ErrorReport by running the error analyzer on the model. """ for config in self._ea_config_list: if config.is_computed: continue config.is_computed = True max_depth = config.max_depth num_leaves = config.num_leaves filter_features = config.filter_features report = self._analyzer.create_error_report(filter_features, max_depth=max_depth, num_leaves=num_leaves) self._ea_report_list.append(report) def get(self): """Get the computed error reports. Must be called after add and compute methods. :return: The computed error reports. :rtype: list[erroranalysis._internal.error_report.ErrorReport] """ return self._ea_report_list def list(self): """List information about the ErrorAnalysisManager. :return: A dictionary of properties. :rtype: dict """ props = {ListProperties.MANAGER_TYPE: self.name} reports = [] for config in self._ea_config_list: report = {} report[Keys.IS_COMPUTED] = config.is_computed report[Keys.MAX_DEPTH] = config.max_depth report[Keys.NUM_LEAVES] = config.num_leaves report[Keys.FILTER_FEATURES] = config.filter_features reports.append(report) props[Keys.REPORTS] = reports return props def get_data(self): """Get error analysis data :return: A array of ErrorAnalysisConfig. :rtype: List[ErrorAnalysisConfig] """ return [self._get_error_analysis(i) for i in self.list()["reports"]] def _get_error_analysis(self, report): error_analysis = ErrorAnalysisData() error_analysis.maxDepth = report[Keys.MAX_DEPTH] error_analysis.numLeaves = report[Keys.NUM_LEAVES] return error_analysis @property def name(self): """Get the name of the error analysis manager. :return: The name of the error analysis manager. :rtype: str """ return ManagerNames.ERROR_ANALYSIS def _save(self, path): """Save the ErrorAnalysisManager to the given path. :param path: The directory path to save the ErrorAnalysisManager to. :type path: str """ top_dir = Path(path) top_dir.mkdir(parents=True, exist_ok=True) # save the reports reports_path = top_dir / REPORTS with open(reports_path, 'w') as file: json.dump(self._ea_report_list, file, default=report_json_converter) # save the configs config_path = top_dir / CONFIG with open(config_path, 'w') as file: json.dump(self._ea_config_list, file, default=config_json_converter) @staticmethod def _load(path, model_analysis): """Load the ErrorAnalysisManager from the given path. :param path: The directory path to load the ErrorAnalysisManager from. :type path: str :param model_analysis: The loaded parent ModelAnalysis. :type model_analysis: ModelAnalysis """ # create the ErrorAnalysisManager without any properties using # the __new__ function, similar to pickle inst = ErrorAnalysisManager.__new__(ErrorAnalysisManager) top_dir = Path(path) reports_path = top_dir / REPORTS with open(reports_path, 'r') as file: ea_report_list = json.load(file, object_hook=as_error_report) inst.__dict__['_ea_report_list'] = ea_report_list config_path = top_dir / CONFIG with open(config_path, 'r') as file: ea_config_list = json.load(file, object_hook=as_error_config) inst.__dict__['_ea_config_list'] = ea_config_list categorical_features = model_analysis.categorical_features inst.__dict__['_categorical_features'] = categorical_features target_column = model_analysis.target_column true_y = model_analysis.test[target_column] dataset = model_analysis.test.drop(columns=[target_column]) inst.__dict__['_dataset'] = dataset inst.__dict__['_true_y'] = true_y feature_names = list(dataset.columns) inst.__dict__['_feature_names'] = feature_names inst.__dict__['_analyzer'] = ModelAnalyzer(model_analysis.model, dataset, true_y, feature_names, categorical_features) return inst
def setup_local(self, explanation, model, dataset, true_y, classes, features, categorical_features, true_y_dataset, pred_y, pred_y_dataset, model_task, metric, max_depth, num_leaves, min_child_samples, sample_dataset, model_available): full_dataset = dataset if true_y_dataset is None: full_true_y = true_y else: full_true_y = true_y_dataset if pred_y_dataset is None: full_pred_y = pred_y else: full_pred_y = pred_y_dataset has_explanation = explanation is not None probability_y = None if has_explanation: if classes is None: has_classes_attr = hasattr(explanation, 'classes') if has_classes_attr and explanation.classes is not None: classes = explanation.classes dataset, true_y = self.input_explanation(explanation, dataset, true_y) row_length = len(dataset) # Only check dataset on explanation for row length bounds if row_length > 100000: raise ValueError("Exceeds maximum number of rows" "for visualization (100000)") elif sample_dataset is not None: dataset = sample_dataset if isinstance(dataset, pd.DataFrame) and hasattr(dataset, 'columns'): self._dataframeColumns = dataset.columns self._dfdtypes = dataset.dtypes try: list_dataset = convert_to_list(dataset) except Exception as ex: ex_str = _format_exception(ex) raise ValueError( "Unsupported dataset type, inner error: {}".format(ex_str)) if has_explanation: self.input_explanation_data(list_dataset, classes) if features is None and hasattr(explanation, 'features'): features = explanation.features if model_available: predicted_y = self.compute_predicted_y(model, dataset) else: predicted_y = self.predicted_y_to_list(pred_y) self.setup_visualization_input(classes, predicted_y, list_dataset, true_y, features) if model_available and is_classifier(model) and \ dataset is not None: try: probability_y = model.predict_proba(dataset) except Exception as ex: ex_str = _format_exception(ex) raise ValueError("Model does not support predict_proba method" " for given dataset type," " inner error: {}".format(ex_str)) try: probability_y = convert_to_list(probability_y) except Exception as ex: ex_str = _format_exception(ex) raise ValueError( "Model predict_proba output of unsupported type," "inner error: {}".format(ex_str)) self.dashboard_input[ ExplanationDashboardInterface.PROBABILITY_Y] = probability_y if model_available: self._error_analyzer = ModelAnalyzer(model, full_dataset, full_true_y, features, categorical_features, model_task, metric, classes) else: # Model task cannot be unknown when passing predictions # Assume classification for backwards compatibility if model_task == ModelTask.UNKNOWN: model_task = ModelTask.CLASSIFICATION self._error_analyzer = PredictionsAnalyzer( full_pred_y, full_dataset, full_true_y, features, categorical_features, model_task, metric, classes) if self._categorical_features: self.dashboard_input[ExplanationDashboardInterface. CATEGORICAL_MAP] = serialize_json_safe( self._error_analyzer.category_dictionary) # Compute metrics on all data cohort if self._error_analyzer.model_task == ModelTask.CLASSIFICATION: if self._error_analyzer.metric is None: metric = Metrics.ERROR_RATE else: metric = self._error_analyzer.metric else: if self._error_analyzer.metric is None: metric = Metrics.MEAN_SQUARED_ERROR else: metric = self._error_analyzer.metric if model_available: full_pred_y = self.compute_predicted_y(model, full_dataset) # If we don't have an explanation or model/probabilities specified # we can try to use model task to figure out the method if not has_explanation and probability_y is None: method = MethodConstants.REGRESSION if self._error_analyzer.model_task == ModelTask.CLASSIFICATION: if (len(np.unique(predicted_y)) > 2): method = MethodConstants.MULTICLASS else: method = MethodConstants.BINARY self.dashboard_input[ ErrorAnalysisDashboardInterface.METHOD] = method
def __init__(self, explanation, model, dataset, true_y, classes, features, categorical_features, true_y_dataset, pred_y, model_task, metric, max_depth, num_leaves): """Initialize the ErrorAnalysis Dashboard Input. :param explanation: An object that represents an explanation. :type explanation: ExplanationMixin :param model: An object that represents a model. It is assumed that for the classification case it has a method of predict_proba() returning the prediction probabilities for each class and for the regression case a method of predict() returning the prediction value. :type model: object :param dataset: A matrix of feature vector examples (# examples x # features), the same samples used to build the explanation. Will overwrite any set on explanation object already. Must have fewer than 10000 rows and fewer than 1000 columns. :type dataset: numpy.array or list[][] or pandas.DataFrame :param true_y: The true labels for the provided explanation. Will overwrite any set on explanation object already. :type true_y: numpy.array or list[] :param classes: The class names. :type classes: numpy.array or list[] :param features: Feature names. :type features: numpy.array or list[] :param categorical_features: The categorical feature names. :type categorical_features: list[str] :param true_y_dataset: The true labels for the provided dataset. Only needed if the explanation has a sample of instances from the original dataset. Otherwise specify true_y parameter only. :type true_y_dataset: numpy.array or list[] :param pred_y: The predicted y values, can be passed in as an alternative to the model and explanation for a more limited view. :type pred_y: numpy.ndarray or list[] :param model_task: Optional parameter to specify whether the model is a classification or regression model. In most cases, the type of the model can be inferred based on the shape of the output, where a classifier has a predict_proba method and outputs a 2 dimensional array, while a regressor has a predict method and outputs a 1 dimensional array. :type model_task: str :param metric: The metric name to evaluate at each tree node or heatmap grid. Currently supported classification metrics include 'error_rate', 'recall_score', 'precision_score', 'f1_score', and 'accuracy_score'. Supported regression metrics include 'mean_absolute_error', 'mean_squared_error', 'r2_score', and 'median_absolute_error'. :type metric: str :param max_depth: The maximum depth of the surrogate tree trained on errors. :type max_depth: int :param num_leaves: The number of leaves of the surrogate tree trained on errors. :type num_leaves: int """ self._model = model full_dataset = dataset if true_y_dataset is None: full_true_y = true_y else: full_true_y = true_y_dataset self._categorical_features = categorical_features self._string_ind_data = None self._categories = [] self._categorical_indexes = [] self._is_classifier = model is not None\ and hasattr(model, SKLearn.PREDICT_PROBA) and \ model.predict_proba is not None self._dataframeColumns = None self.dashboard_input = {} has_explanation = explanation is not None feature_length = None self._max_depth = max_depth self._num_leaves = num_leaves if has_explanation: if classes is None: has_classes_attr = hasattr(explanation, 'classes') if has_classes_attr and explanation.classes is not None: classes = explanation.classes dataset, true_y = self.input_explanation(explanation, dataset, true_y) row_length = len(dataset) # Only check dataset on explanation for row length bounds if row_length > 100000: raise ValueError("Exceeds maximum number of rows" "for visualization (100000)") if classes is not None: classes = self._convert_to_list(classes) self.dashboard_input[ ExplanationDashboardInterface.CLASS_NAMES] = classes class_to_index = {k: v for v, k in enumerate(classes)} if isinstance(dataset, pd.DataFrame) and hasattr(dataset, 'columns'): self._dataframeColumns = dataset.columns self._dfdtypes = dataset.dtypes try: list_dataset = self._convert_to_list(dataset) except Exception as ex: ex_str = _format_exception(ex) raise ValueError( "Unsupported dataset type, inner error: {}".format(ex_str)) if has_explanation: self.input_explanation_data(explanation, list_dataset, classes) if features is None and hasattr(explanation, 'features'): features = explanation.features model_available = model is not None if model_available and pred_y is not None: raise ValueError('Only model or pred_y can be specified, not both') self.dashboard_input[ENABLE_PREDICT] = model_available if model_available: predicted_y = self.compute_predicted_y(model, dataset) else: predicted_y = self.predicted_y_to_list(pred_y) if predicted_y is not None: # If classes specified, convert predicted_y to # numeric representation if classes is not None and predicted_y[0] in class_to_index: for i in range(len(predicted_y)): predicted_y[i] = class_to_index[predicted_y[i]] self.dashboard_input[ ExplanationDashboardInterface.PREDICTED_Y] = predicted_y row_length = 0 if list_dataset is not None: row_length, feature_length = np.shape(list_dataset) if feature_length > 1000: raise ValueError("Exceeds maximum number of features for" " visualization (1000). Please regenerate the" " explanation using fewer features or" " initialize the dashboard without passing a" " dataset.") self.dashboard_input[ExplanationDashboardInterface. TRAINING_DATA] = serialize_json_safe( list_dataset) self.dashboard_input[ExplanationDashboardInterface. IS_CLASSIFIER] = self._is_classifier if true_y is not None and len(true_y) == row_length: list_true_y = self._convert_to_list(true_y) # If classes specified, convert true_y to numeric representation if classes is not None and list_true_y[0] in class_to_index: for i in range(len(list_true_y)): list_true_y[i] = class_to_index[list_true_y[i]] self.dashboard_input[ ExplanationDashboardInterface.TRUE_Y] = list_true_y if features is not None: features = self._convert_to_list(features) if feature_length is not None and len(features) != feature_length: raise ValueError("Feature vector length mismatch:" " feature names length differs" " from local explanations dimension") self.dashboard_input[FEATURE_NAMES] = features if model_available and hasattr(model, SKLearn.PREDICT_PROBA) \ and model.predict_proba is not None and dataset is not None: try: probability_y = model.predict_proba(dataset) except Exception as ex: ex_str = _format_exception(ex) raise ValueError("Model does not support predict_proba method" " for given dataset type," " inner error: {}".format(ex_str)) try: probability_y = self._convert_to_list(probability_y) except Exception as ex: ex_str = _format_exception(ex) raise ValueError( "Model predict_proba output of unsupported type," "inner error: {}".format(ex_str)) self.dashboard_input[ ExplanationDashboardInterface.PROBABILITY_Y] = probability_y if model_available: self._error_analyzer = ModelAnalyzer(model, full_dataset, full_true_y, features, categorical_features, model_task, metric) else: # Model task cannot be unknown when passing predictions # Assume classification for backwards compatibility if model_task == ModelTask.UNKNOWN: model_task = ModelTask.CLASSIFICATION self._error_analyzer = PredictionsAnalyzer(pred_y, full_dataset, full_true_y, features, categorical_features, model_task, metric) if self._categorical_features: self.dashboard_input[ ExplanationDashboardInterface. CATEGORICAL_MAP] = self._error_analyzer.category_dictionary # Compute metrics on all data cohort if self._error_analyzer.model_task == ModelTask.CLASSIFICATION: if self._error_analyzer.metric is None: metric = Metrics.ERROR_RATE else: metric = self._error_analyzer.metric else: if self._error_analyzer.metric is None: metric = Metrics.MEAN_SQUARED_ERROR else: metric = self._error_analyzer.metric if model_available and true_y_dataset is not None: full_predicted_y = self.compute_predicted_y(model, full_dataset) else: full_predicted_y = predicted_y self.set_root_metric(full_predicted_y, full_true_y, metric)
class ErrorAnalysisManager(BaseManager): """Defines the ErrorAnalysisManager for discovering errors in a model.""" def __init__(self, model: Any, dataset: pd.DataFrame, target_column: str, classes: Optional[List] = None, categorical_features: Optional[List[str]] = None): """Creates an ErrorAnalysisManager object. :param model: The model to analyze errors on. A model that implements sklearn.predict or sklearn.predict_proba or function that accepts a 2d ndarray. :type model: object :param dataset: The dataset including the label column. :type dataset: pandas.DataFrame :param target_column: The name of the label column. :type target_column: str :param classes: Class names as a list of strings. The order of the class names should match that of the model output. Only required if analyzing a classifier. :type classes: list :param categorical_features: The categorical feature names. :type categorical_features: list[str] """ self._true_y = dataset[target_column] self._dataset = dataset.drop(columns=[target_column]) self._feature_names = list(self._dataset.columns) self._classes = classes self._categorical_features = categorical_features self._ea_config_list = [] self._ea_report_list = [] self._analyzer = ModelAnalyzer(model, self._dataset, self._true_y, self._feature_names, self._categorical_features, classes=self._classes) def add(self, max_depth: int = 3, num_leaves: int = 31, min_child_samples: int = 20, filter_features: Optional[List] = None): """Add an error analyzer to be computed later. :param max_depth: The maximum depth of the tree. :type max_depth: Optional[int] :param num_leaves: The number of leaves in the tree. :type num_leaves: Optional[int] :param min_child_samples: The minimal number of data required to create one leaf. :type min_child_samples: Optional[int] :param filter_features: One or two features to use for the matrix filter. :type filter_features: Optional[list] """ if self._analyzer.model is None: raise UserConfigValidationException( 'Model is required for error analysis') ea_config = ErrorAnalysisConfig(max_depth=max_depth, num_leaves=num_leaves, min_child_samples=min_child_samples, filter_features=filter_features) is_duplicate = ea_config.is_duplicate(self._ea_config_list) if is_duplicate: raise DuplicateManagerConfigException( "Duplicate config specified for error analysis," "config already added") else: self._ea_config_list.append(ea_config) def compute(self): """Creates an ErrorReport by running the error analyzer on the model. """ for config in self._ea_config_list: if config.is_computed: continue config.is_computed = True max_depth = config.max_depth num_leaves = config.num_leaves min_child_samples = config.min_child_samples filter_features = config.filter_features report = self._analyzer.create_error_report( filter_features, max_depth=max_depth, min_child_samples=min_child_samples, num_leaves=num_leaves, compute_importances=True, compute_root_stats=True) # Validate the serialized output against schema schema = ErrorAnalysisManager._get_error_analysis_schema() jsonschema.validate(json.loads(report.to_json()), schema) self._ea_report_list.append(report) def get(self): """Get the computed error reports. Must be called after add and compute methods. :return: The computed error reports. :rtype: list[erroranalysis._internal.error_report.ErrorReport] """ return self._ea_report_list @staticmethod def _get_error_analysis_schema(): """Get the schema for validating the error analysis output.""" schema_directory = (Path(__file__).parent.parent / '_tools' / 'error_analysis' / 'dashboard_schemas') schema_filename = 'error_analysis_output_v0.0.json' schema_filepath = schema_directory / schema_filename with open(schema_filepath, 'r') as f: return json.load(f) def list(self): """List information about the ErrorAnalysisManager. :return: A dictionary of properties. :rtype: dict """ props = {ListProperties.MANAGER_TYPE: self.name} reports = [] for config in self._ea_config_list: report = {} report[Keys.IS_COMPUTED] = config.is_computed report[Keys.MAX_DEPTH] = config.max_depth report[Keys.NUM_LEAVES] = config.num_leaves report[Keys.MIN_CHILD_SAMPLES] = config.min_child_samples report[Keys.FILTER_FEATURES] = config.filter_features reports.append(report) props[Keys.REPORTS] = reports return props def get_data(self): """Get error analysis data :return: A array of ErrorAnalysisConfig. :rtype: List[ErrorAnalysisConfig] """ report_props = zip(self.get(), self.list()[Keys.REPORTS]) return [ self._get_error_analysis(report, props) for report, props in report_props ] def _get_error_analysis(self, report, props): error_analysis = ErrorAnalysisData() error_analysis.maxDepth = props[Keys.MAX_DEPTH] error_analysis.numLeaves = props[Keys.NUM_LEAVES] error_analysis.minChildSamples = props[Keys.MIN_CHILD_SAMPLES] error_analysis.tree = report.tree error_analysis.matrix = report.matrix error_analysis.importances = report.importances error_analysis.metric = metric_to_display_name[self._analyzer.metric] error_analysis.root_stats = report.root_stats return error_analysis @property def name(self): """Get the name of the error analysis manager. :return: The name of the error analysis manager. :rtype: str """ return ManagerNames.ERROR_ANALYSIS def _save(self, path): """Save the ErrorAnalysisManager to the given path. :param path: The directory path to save the ErrorAnalysisManager to. :type path: str """ top_dir = Path(path) top_dir.mkdir(parents=True, exist_ok=True) if len(self._ea_config_list) != len(self._ea_report_list): raise ConfigAndResultMismatchException( "The number of error analysis configs {0} doesn't match the " "number of results {1}".format(len(self._ea_config_list), len(self._ea_report_list))) for index in range(0, len(self._ea_report_list)): # save the configs directory_manager = DirectoryManager(parent_directory_path=path) config_path = (directory_manager.create_config_directory() / 'config.json') ea_config = self._ea_config_list[index] with open(config_path, 'w') as file: json.dump(ea_config, file, default=config_json_converter) # save the reports report_path = (directory_manager.create_data_directory() / 'report.json') ea_report = self._ea_report_list[index] with open(report_path, 'w') as file: json.dump(ea_report, file, default=report_json_converter) @staticmethod def _load(path, rai_insights): """Load the ErrorAnalysisManager from the given path. :param path: The directory path to load the ErrorAnalysisManager from. :type path: str :param rai_insights: The loaded parent RAIInsights. :type rai_insights: RAIInsights :return: The ErrorAnalysisManager manager after loading. :rtype: ErrorAnalysisManager """ # create the ErrorAnalysisManager without any properties using # the __new__ function, similar to pickle inst = ErrorAnalysisManager.__new__(ErrorAnalysisManager) ea_config_list = [] ea_report_list = [] all_ea_dirs = DirectoryManager.list_sub_directories(path) for ea_dir in all_ea_dirs: directory_manager = DirectoryManager(parent_directory_path=path, sub_directory_name=ea_dir) config_path = (directory_manager.get_config_directory() / 'config.json') with open(config_path, 'r') as file: ea_config = json.load(file, object_hook=as_error_config) ea_config_list.append(ea_config) report_path = (directory_manager.get_data_directory() / 'report.json') with open(report_path, 'r') as file: ea_report = json.load(file, object_hook=as_error_report) # Validate the serialized output against schema schema = ErrorAnalysisManager._get_error_analysis_schema() jsonschema.validate(json.loads(ea_report.to_json()), schema) ea_report_list.append(ea_report) inst.__dict__['_ea_report_list'] = ea_report_list inst.__dict__['_ea_config_list'] = ea_config_list categorical_features = rai_insights.categorical_features inst.__dict__['_categorical_features'] = categorical_features target_column = rai_insights.target_column true_y = rai_insights.test[target_column] dataset = rai_insights.test.drop(columns=[target_column]) inst.__dict__['_dataset'] = dataset inst.__dict__['_true_y'] = true_y feature_names = list(dataset.columns) inst.__dict__['_feature_names'] = feature_names inst.__dict__['_analyzer'] = ModelAnalyzer(rai_insights.model, dataset, true_y, feature_names, categorical_features) return inst
def __init__(self, explanation, model, dataset, true_y, classes, features, categorical_features, true_y_dataset, pred_y): """Initialize the ErrorAnalysis Dashboard Input. :param explanation: An object that represents an explanation. :type explanation: ExplanationMixin :param model: An object that represents a model. It is assumed that for the classification case it has a method of predict_proba() returning the prediction probabilities for each class and for the regression case a method of predict() returning the prediction value. :type model: object :param dataset: A matrix of feature vector examples (# examples x # features), the same samples used to build the explanation. Will overwrite any set on explanation object already. Must have fewer than 10000 rows and fewer than 1000 columns. :type dataset: numpy.array or list[][] or pandas.DataFrame :param true_y: The true labels for the provided explanation. Will overwrite any set on explanation object already. :type true_y: numpy.array or list[] :param classes: The class names. :type classes: numpy.array or list[] :param features: Feature names. :type features: numpy.array or list[] :param categorical_features: The categorical feature names. :type categorical_features: list[str] :param true_y_dataset: The true labels for the provided dataset. Only needed if the explanation has a sample of instances from the original dataset. Otherwise specify true_y parameter only. :type true_y_dataset: numpy.array or list[] :param pred_y: The predicted y values, can be passed in as an alternative to the model and explanation for a more limited view. :type pred_y: numpy.ndarray or list[] """ self._model = model full_dataset = dataset if true_y_dataset is None: full_true_y = true_y else: full_true_y = true_y_dataset self._categorical_features = categorical_features self._string_ind_data = None self._categories = [] self._categorical_indexes = [] self._is_classifier = model is not None\ and hasattr(model, SKLearn.PREDICT_PROBA) and \ model.predict_proba is not None self._dataframeColumns = None self.dashboard_input = {} has_explanation = explanation is not None feature_length = None if has_explanation: if classes is None: has_classes_attr = hasattr(explanation, 'classes') if has_classes_attr and explanation.classes is not None: classes = explanation.classes dataset, true_y = self.input_explanation(explanation, dataset, true_y) if classes is not None: classes = self._convert_to_list(classes) self.dashboard_input[ ExplanationDashboardInterface.CLASS_NAMES] = classes class_to_index = {k: v for v, k in enumerate(classes)} if isinstance(dataset, pd.DataFrame) and hasattr(dataset, 'columns'): self._dataframeColumns = dataset.columns try: list_dataset = self._convert_to_list(dataset) except Exception as ex: ex_str = _format_exception(ex) raise ValueError( "Unsupported dataset type, inner error: {}".format(ex_str)) if has_explanation: self.input_explanation_data(explanation, list_dataset, classes) if features is None and hasattr(explanation, 'features'): features = explanation.features model_available = model is not None if model_available and pred_y is not None: raise ValueError('Only model or pred_y can be specified, not both') self.dashboard_input[ENABLE_PREDICT] = model_available if model_available: predicted_y = self.compute_predicted_y(model, dataset) else: predicted_y = self.predicted_y_to_list(pred_y) if predicted_y is not None: # If classes specified, convert predicted_y to # numeric representation if classes is not None and predicted_y[0] in class_to_index: for i in range(len(predicted_y)): predicted_y[i] = class_to_index[predicted_y[i]] self.dashboard_input[ ExplanationDashboardInterface.PREDICTED_Y] = predicted_y row_length = 0 if list_dataset is not None: row_length, feature_length = np.shape(list_dataset) if row_length > 100000: raise ValueError("Exceeds maximum number of rows" "for visualization (100000)") if feature_length > 1000: raise ValueError("Exceeds maximum number of features for" " visualization (1000). Please regenerate the" " explanation using fewer features or" " initialize the dashboard without passing a" " dataset.") self.dashboard_input[ExplanationDashboardInterface. TRAINING_DATA] = _serialize_json_safe( list_dataset) self.dashboard_input[ExplanationDashboardInterface. IS_CLASSIFIER] = self._is_classifier if true_y is not None and len(true_y) == row_length: list_true_y = self._convert_to_list(true_y) # If classes specified, convert true_y to numeric representation if classes is not None and list_true_y[0] in class_to_index: for i in range(len(list_true_y)): list_true_y[i] = class_to_index[list_true_y[i]] self.dashboard_input[ ExplanationDashboardInterface.TRUE_Y] = list_true_y if features is not None: features = self._convert_to_list(features) if feature_length is not None and len(features) != feature_length: raise ValueError("Feature vector length mismatch:" " feature names length differs" " from local explanations dimension") self.dashboard_input[FEATURE_NAMES] = features if model_available and hasattr(model, SKLearn.PREDICT_PROBA) \ and model.predict_proba is not None and dataset is not None: try: probability_y = model.predict_proba(dataset) except Exception as ex: ex_str = _format_exception(ex) raise ValueError("Model does not support predict_proba method" " for given dataset type," " inner error: {}".format(ex_str)) try: probability_y = self._convert_to_list(probability_y) except Exception as ex: ex_str = _format_exception(ex) raise ValueError( "Model predict_proba output of unsupported type," "inner error: {}".format(ex_str)) self.dashboard_input[ ExplanationDashboardInterface.PROBABILITY_Y] = probability_y if model_available: self._error_analyzer = ModelAnalyzer(model, full_dataset, full_true_y, features, categorical_features) else: self._error_analyzer = PredictionsAnalyzer(pred_y, full_dataset, full_true_y, features, categorical_features) if self._categorical_features: self.dashboard_input[ ExplanationDashboardInterface. CATEGORICAL_MAP] = self._error_analyzer.category_dictionary
class ModelAnalysisDashboardInput: def __init__(self, analysis: ModelAnalysis): """Initialize the Explanation Dashboard Input. :param analysis: An ModelAnalysis object that represents an explanation. :type analysis: ModelAnalysis """ self._analysis = analysis model = analysis.model self._is_classifier = model is not None\ and hasattr(model, SKLearn.PREDICT_PROBA) and \ model.predict_proba is not None self._dataframeColumns = None self.dashboard_input = ModelAnalysisDashboardData() self.dashboard_input.dataset = self._get_dataset() self._feature_length = len(self.dashboard_input.dataset.featureNames) self._row_length = len(self.dashboard_input.dataset.features) self.dashboard_input.modelExplanationData = [ self._get_interpret(i) for i in self._analysis.explainer.get() ] self.dashboard_input.errorAnalysisConfig = [ self._get_error_analysis(i) for i in self._analysis.error_analysis.list()["reports"] ] x_test = analysis.test.drop(columns=[analysis.target_column]) y_test = analysis.test[analysis.target_column] self._error_analyzer = ModelAnalyzer(model, x_test, y_test, x_test.columns.values.tolist(), analysis.categorical_features) def on_predict(self, data): try: if self._dataframeColumns is not None: data = pd.DataFrame(data, columns=self._dataframeColumns) if (self._is_classifier): prediction = self._convert_to_list( self._analysis.model.predict_proba(data)) else: prediction = self._convert_to_list( self._analysis.model.predict(data)) return {WidgetRequestResponseConstants.data: prediction} except Exception as e: print(e) traceback.print_exc() return { WidgetRequestResponseConstants.error: "Model threw exception" " while predicting...", WidgetRequestResponseConstants.data: [] } def debug_ml(self, data): try: features, filters, composite_filters, max_depth, num_leaves = data json_tree = self._error_analyzer.compute_error_tree( features, filters, composite_filters, max_depth, num_leaves) return {WidgetRequestResponseConstants.data: json_tree} except Exception as e: print(e) traceback.print_exc() return { WidgetRequestResponseConstants.error: "Failed to generate json tree representation", WidgetRequestResponseConstants.data: [] } def matrix(self, data): try: features, filters, composite_filters = data if features[0] is None and features[1] is None: return {WidgetRequestResponseConstants.data: []} json_matrix = self._error_analyzer.compute_matrix( features, filters, composite_filters) return {WidgetRequestResponseConstants.data: json_matrix} except Exception as e: print(e) traceback.print_exc() return { WidgetRequestResponseConstants.error: "Failed to generate json matrix representation", WidgetRequestResponseConstants.data: [] } def importances(self): try: scores = self._error_analyzer.compute_importances() return {WidgetRequestResponseConstants.data: scores} except Exception as e: print(e) traceback.print_exc() return { WidgetRequestResponseConstants.error: "Failed to generate feature importances", WidgetRequestResponseConstants.data: [] } def _get_dataset(self): dashboard_dataset = Dataset() dashboard_dataset.classNames = self._convert_to_list( self._analysis._classes) predicted_y = None feature_length = None dataset: pd.DataFrame = self._analysis.test.drop( [self._analysis.target_column], axis=1) if isinstance(dataset, pd.DataFrame) and hasattr(dataset, 'columns'): self._dataframeColumns = dataset.columns try: list_dataset = self._convert_to_list(dataset) except Exception as ex: ex_str = _format_exception(ex) raise ValueError( "Unsupported dataset type, inner error: {}".format(ex_str)) if dataset is not None and self._analysis.model is not None: try: predicted_y = self._analysis.model.predict(dataset) except Exception as ex: ex_str = _format_exception(ex) msg = "Model does not support predict method for given" "dataset type, inner error: {}".format(ex_str) raise ValueError(msg) try: predicted_y = self._convert_to_list(predicted_y) except Exception as ex: ex_str = _format_exception(ex) raise ValueError("Model prediction output of unsupported type," "inner error: {}".format(ex_str)) if predicted_y is not None: if (self._analysis.task_type == "classification" and dashboard_dataset.classNames is not None): predicted_y = [ dashboard_dataset.classNames.index(y) for y in predicted_y ] dashboard_dataset.predictedY = predicted_y row_length = 0 if list_dataset is not None: row_length, feature_length = np.shape(list_dataset) if row_length > 100000: raise ValueError("Exceeds maximum number of rows" "for visualization (100000)") if feature_length > 1000: raise ValueError("Exceeds maximum number of features for" " visualization (1000). Please regenerate the" " explanation using fewer features or" " initialize the dashboard without passing a" " dataset.") dashboard_dataset.features = _serialize_json_safe(list_dataset) true_y = self._analysis.test[self._analysis.target_column] if true_y is not None and len(true_y) == row_length: if (self._analysis.task_type == "classification" and dashboard_dataset.classNames is not None): true_y = [ dashboard_dataset.classNames.index(y) for y in true_y ] dashboard_dataset.trueY = self._convert_to_list(true_y) features = dataset.columns if features is not None: features = self._convert_to_list(features) if feature_length is not None and len(features) != feature_length: raise ValueError("Feature vector length mismatch:" " feature names length differs" " from local explanations dimension") dashboard_dataset.featureNames = features if (self._analysis.model is not None and hasattr(self._analysis.model, SKLearn.PREDICT_PROBA) and self._analysis.model.predict_proba is not None and dataset is not None): try: probability_y = self._analysis.model.predict_proba(dataset) except Exception as ex: ex_str = _format_exception(ex) raise ValueError("Model does not support predict_proba method" " for given dataset type," " inner error: {}".format(ex_str)) try: probability_y = self._convert_to_list(probability_y) except Exception as ex: ex_str = _format_exception(ex) raise ValueError( "Model predict_proba output of unsupported type," "inner error: {}".format(ex_str)) dashboard_dataset.probabilityY = probability_y return dashboard_dataset def _get_interpret(self, explanation): interpretation = ModelExplanationData() # List of explanations, key of explanation type is "explanation_type" if explanation is not None: mli_explanations = explanation.data(-1)["mli"] else: mli_explanations = None local_explanation = self._find_first_explanation( ExplanationDashboardInterface.MLI_LOCAL_EXPLANATION_KEY, mli_explanations) global_explanation = self._find_first_explanation( ExplanationDashboardInterface.MLI_GLOBAL_EXPLANATION_KEY, mli_explanations) ebm_explanation = self._find_first_explanation( ExplanationDashboardInterface.MLI_EBM_GLOBAL_EXPLANATION_KEY, mli_explanations) if explanation is not None and hasattr(explanation, 'method'): interpretation.method = explanation.method local_dim = None if local_explanation is not None or global_explanation is not None\ or ebm_explanation is not None: interpretation.precomputedExplanations = PrecomputedExplanations() if local_explanation is not None: try: local_feature_importance = FeatureImportance() local_feature_importance.scores = self._convert_to_list( local_explanation["scores"]) if np.shape(local_feature_importance.scores)[-1] > 1000: raise ValueError("Exceeds maximum number of features for " "visualization (1000). Please regenerate" " the explanation using fewer features.") local_feature_importance.intercept = self._convert_to_list( local_explanation["intercept"]) # We can ignore perf explanation data. # Note if it is added back at any point, # the numpy values will need to be converted to python, # otherwise serialization fails. local_explanation["perf"] = None interpretation.precomputedExplanations.localFeatureImportance\ = local_feature_importance except Exception as ex: ex_str = _format_exception(ex) raise ValueError("Unsupported local explanation type," "inner error: {}".format(ex_str)) if self._analysis.test is not None: local_dim = np.shape(local_feature_importance.scores) if len(local_dim) != 2 and len(local_dim) != 3: raise ValueError( "Local explanation expected to be a 2D or 3D list") if (len(local_dim) == 2 and (local_dim[1] != self._feature_length or local_dim[0] != self._row_length)): raise ValueError("Shape mismatch: local explanation" "length differs from dataset") if (len(local_dim) == 3 and (local_dim[2] != self._feature_length or local_dim[1] != self._row_length)): raise ValueError("Shape mismatch: local explanation" " length differs from dataset") if global_explanation is not None: try: global_feature_importance = FeatureImportance() global_feature_importance.scores = self._convert_to_list( global_explanation["scores"]) if 'intercept' in global_explanation: global_feature_importance.intercept\ = self._convert_to_list( global_explanation["intercept"]) interpretation.precomputedExplanations.globalFeatureImportance\ = global_explanation except Exception as ex: ex_str = _format_exception(ex) raise ValueError("Unsupported global explanation type," "inner error: {}".format(ex_str)) if ebm_explanation is not None: try: ebm_feature_importance = EBMGlobalExplanation() ebm_feature_importance.feature_list\ = ebm_explanation["feature_list"] interpretation.precomputedExplanations.ebmGlobalExplanation\ = ebm_feature_importance except Exception as ex: ex_str = _format_exception(ex) raise ValueError( "Unsupported ebm explanation type: {}".format(ex_str)) return interpretation def _get_error_analysis(self, report): error_analysis = ErrorAnalysisConfig() error_analysis.maxDepth = report[ErrorAnalysisManagerKeys.MAX_DEPTH] error_analysis.numLeaves = report[ErrorAnalysisManagerKeys.NUM_LEAVES] return error_analysis def _convert_to_list(self, array): if issparse(array): if array.shape[1] > 1000: raise ValueError("Exceeds maximum number of features" " for visualization (1000). Please regenerate" " the explanation using fewer features" " or initialize the dashboard without passing" " a dataset.") return array.toarray().tolist() if (isinstance(array, pd.DataFrame)): return array.values.tolist() if (isinstance(array, pd.Series)): return array.values.tolist() if (isinstance(array, np.ndarray)): return array.tolist() if (isinstance(array, pd.Index)): return array.tolist() return array def _find_first_explanation(self, key, mli_explanations): if mli_explanations is None: return None new_array = [ explanation for explanation in mli_explanations if explanation[ ExplanationDashboardInterface.MLI_EXPLANATION_TYPE_KEY] == key ] if len(new_array) > 0: return new_array[0]["value"] return None