def compute_matrix(analyzer, features, filters, composite_filters, quantile_binning=False, num_bins=BIN_THRESHOLD): if num_bins <= 0: raise ValueError( 'Number of bins parameter must be greater than 0 for the heatmap') if features[0] is None and features[1] is None: raise ValueError( 'One or two features must be specified to compute the heat map') filtered_df = filter_from_cohort(analyzer, filters, composite_filters) true_y = filtered_df[TRUE_Y] dropped_cols = [TRUE_Y, ROW_INDEX] is_model_analyzer = hasattr(analyzer, 'model') if not is_model_analyzer: pred_y = filtered_df[PRED_Y] dropped_cols.append(PRED_Y) input_data = filtered_df.drop(columns=dropped_cols) is_pandas = isinstance(analyzer.dataset, pd.DataFrame) metric = analyzer.metric if is_pandas: true_y = true_y.to_numpy() else: input_data = input_data.to_numpy() if is_model_analyzer: pred_y = analyzer.model.predict(input_data) if is_model_analyzer: if analyzer.model_task == ModelTask.CLASSIFICATION: diff = analyzer.model.predict(input_data) != true_y else: diff = analyzer.model.predict(input_data) - true_y else: if analyzer.model_task == ModelTask.CLASSIFICATION: diff = pred_y != true_y else: diff = pred_y - true_y if not isinstance(diff, np.ndarray): diff = np.array(diff) if not isinstance(pred_y, np.ndarray): pred_y = np.array(pred_y) if not isinstance(true_y, np.ndarray): true_y = np.array(true_y) indexes = [] for feature in features: if feature is None: continue indexes.append(analyzer.feature_names.index(feature)) if is_pandas: input_data = input_data.to_numpy() dataset_sub_features = input_data[:, indexes] dataset_sub_names = np.array(analyzer.feature_names)[np.array(indexes)] df = pd.DataFrame(dataset_sub_features, columns=dataset_sub_names) df_err = df.copy() df_err[DIFF] = diff if metric == Metrics.ERROR_RATE: df_err = df_err[df_err[DIFF]] else: df_err[TRUE_Y] = true_y df_err[PRED_Y] = pred_y # construct matrix matrix = [] if len(dataset_sub_names) == 2: feat1 = dataset_sub_names[0] feat2 = dataset_sub_names[1] unique_count1 = len(df[feat1].unique()) unique_count2 = len(df[feat2].unique()) f1_is_cat = False f2_is_cat = False if analyzer.categorical_features is not None: f1_is_cat = feat1 in analyzer.categorical_features f2_is_cat = feat2 in analyzer.categorical_features if unique_count1 > num_bins and not f1_is_cat: tabdf1 = bin_data(df, feat1, num_bins, quantile_binning=quantile_binning) categories1 = tabdf1.cat.categories if len(categories1) < num_bins: warn_duplicate_edges(feat1) tabdf1_err = bin_data(df_err, feat1, categories1, quantile_binning=quantile_binning) else: tabdf1 = df[feat1] tabdf1_err = df_err[feat1] categories1 = np.unique(tabdf1.to_numpy(), return_counts=True)[0] if unique_count2 > num_bins and not f2_is_cat: tabdf2 = bin_data(df, feat2, num_bins, quantile_binning=quantile_binning) categories2 = tabdf2.cat.categories if len(categories2) < num_bins: warn_duplicate_edges(feat2) tabdf2_err = bin_data(df_err, feat2, categories2, quantile_binning=quantile_binning) else: tabdf2 = df[feat2] tabdf2_err = df_err[feat2] categories2 = np.unique(tabdf2.to_numpy(), return_counts=True)[0] if metric == Metrics.ERROR_RATE: matrix_total = pd.crosstab(tabdf1, tabdf2, rownames=[feat1], colnames=[feat2]) matrix_error = pd.crosstab(tabdf1_err, tabdf2_err, rownames=[feat1], colnames=[feat2]) else: if is_multi_agg_metric(metric): ordered_labels = get_ordered_classes(analyzer.classes, true_y, pred_y) aggfunc = _MultiMetricAggFunc(metric_to_func[metric], ordered_labels, metric) else: aggfunc = _AggFunc(metric_to_func[metric]) matrix_total = pd.crosstab(tabdf1, tabdf2, rownames=[feat1], colnames=[feat2]) matrix_error = pd.crosstab(tabdf1_err, tabdf2_err, rownames=[feat1], colnames=[feat2], values=list( zip(df_err[TRUE_Y], df_err[PRED_Y])), aggfunc=aggfunc._agg_func_pair) fill_matrix_nulls(matrix_total, aggfunc._fill_na_value()) fill_matrix_nulls(matrix_error, aggfunc._fill_na_value()) matrix = matrix_2d(categories1, categories2, matrix_total, matrix_error, metric) else: feat1 = dataset_sub_names[0] unique_count1 = len(df[feat1].unique()) f1_is_cat = False if analyzer.categorical_features is not None: f1_is_cat = feat1 in analyzer.categorical_features if unique_count1 > num_bins and not f1_is_cat: cutdf = bin_data(df, feat1, num_bins, quantile_binning=quantile_binning) num_categories = len(cutdf.cat.categories) bin_range = range(num_categories) if len(cutdf.cat.categories) < num_bins: warn_duplicate_edges(feat1) catr = cutdf.cat.rename_categories(bin_range) catn, counts = np.unique(catr.to_numpy(), return_counts=True) # fix counts to include skipped categories fix_counts = [] counts_idx = 0 for idx, catdf in enumerate(cutdf.cat.categories): if idx not in catn: fix_counts.append(0) else: fix_counts.append(counts[counts_idx]) counts_idx += 1 counts = fix_counts cut_err = bin_data(df_err, feat1, cutdf.cat.categories, quantile_binning=quantile_binning) catr_err = cut_err.cat.rename_categories(bin_range) val_err, counts_err = np.unique(catr_err.to_numpy(), return_counts=True) val_err = cut_err.cat.categories[val_err] categories = cutdf.cat.categories else: categories, counts = np.unique(df[feat1].to_numpy(), return_counts=True) val_err, counts_err = np.unique(df_err[feat1].to_numpy(), return_counts=True) cut_err = df_err # Compute the given metric for each group, if not using error rate if metric != Metrics.ERROR_RATE: if is_multi_agg_metric(metric): ordered_labels = get_ordered_classes(analyzer.classes, true_y, pred_y) aggfunc = _MultiMetricAggFunc(metric_to_func[metric], ordered_labels, metric) else: aggfunc = _AggFunc(metric_to_func[metric]) cutdf_err = pd.DataFrame(cut_err) cutdf_err['metric_values'] = list( zip(df_err[TRUE_Y], df_err[PRED_Y])) grouped = cutdf_err.groupby([feat1]) agg_func = {'metric_values': aggfunc._agg_func_triplet} counts_err = grouped.agg(agg_func) counts_err = counts_err.values.ravel() matrix = matrix_1d(categories, val_err, counts, counts_err, metric) return matrix
def matrix_1d(categories, values_err, counts, counts_err, metric): matrix = [] matrix_row = [] for col_idx in reversed(range(len(categories))): cat = categories[col_idx] if metric == Metrics.ERROR_RATE: false_count = 0 if cat in values_err: index_err = list(values_err).index(cat) false_count = int(counts_err[index_err]) matrix_row.append({ FALSE_COUNT: false_count, COUNT: int(counts[col_idx]), METRIC_NAME: metric_to_display_name[metric] }) elif is_multi_agg_metric(metric): tp_sum = [] fp_sum = [] fn_sum = [] tn_sum = [] metric_value = 0 error = 0 if cat in values_err: metric_value = float(counts_err[col_idx][0]) tp_sum = counts_err[col_idx][1] fp_sum = counts_err[col_idx][2] fn_sum = counts_err[col_idx][3] tn_sum = counts_err[col_idx][4] error = float(counts_err[col_idx][5]) if math.isnan(metric_value): metric_value = 0.0 matrix_row.append({ METRIC_VALUE: metric_value, TP: tp_sum, FP: fp_sum, FN: fn_sum, TN: tn_sum, ERROR: error, METRIC_NAME: metric_to_display_name[metric], COUNT: int(counts[col_idx]) }) else: metric_value = 0 if cat in values_err: metric_value = float(counts_err[col_idx]) if math.isnan(metric_value): metric_value = 0.0 matrix_row.append({ METRIC_VALUE: metric_value, METRIC_NAME: metric_to_display_name[metric], COUNT: int(counts[col_idx]) }) matrix.append(matrix_row) category = [] category_min_interval = [] category_max_interval = [] for cat in reversed(categories): if isinstance(categories, pd.IntervalIndex): category_min_interval.append(cat.left) category_max_interval.append(cat.right) category.append(str(cat)) else: category.append(get_py_value(cat)) category1 = { VALUES: category, INTERVAL_MIN: category_min_interval, INTERVAL_MAX: category_max_interval } return {MATRIX: matrix, CATEGORY1: category1}
def matrix_2d(categories1, categories2, matrix_counts, matrix_err_counts, metric): matrix = [] category1 = [] category1_min_interval = [] category1_max_interval = [] is_multi_agg = is_multi_agg_metric(metric) for row_index in reversed(range(len(categories1))): matrix_row = [] cat1 = categories1[row_index] if isinstance(categories1, pd.IntervalIndex): category1_min_interval.append(cat1.left) category1_max_interval.append(cat1.right) category1.append(str(cat1)) else: category1.append(get_py_value(cat1)) for col_index in range(len(categories2)): cat2 = categories2[col_index] index_exists_err = cat1 in matrix_err_counts.index col_exists_err = cat2 in matrix_err_counts.columns if metric == Metrics.ERROR_RATE: false_count = 0 if index_exists_err and col_exists_err: false_count = int(matrix_err_counts.loc[cat1, cat2]) elif is_multi_agg: tp_sum = [] fp_sum = [] fn_sum = [] tn_sum = [] metric_value = 0 if index_exists_err and col_exists_err: metric_value = float(matrix_err_counts.loc[cat1, cat2][0]) tp_sum = matrix_err_counts.loc[cat1, cat2][1] fp_sum = matrix_err_counts.loc[cat1, cat2][2] fn_sum = matrix_err_counts.loc[cat1, cat2][3] tn_sum = matrix_err_counts.loc[cat1, cat2][4] error = float(matrix_err_counts.loc[cat1, cat2][5]) else: metric_value = 0 if index_exists_err and col_exists_err: metric_value = float(matrix_err_counts.loc[cat1, cat2]) index_exists = cat1 in matrix_counts.index col_exists = cat2 in matrix_counts.columns total_count = 0 if index_exists and col_exists: total_count = int(matrix_counts.loc[cat1, cat2]) if metric == Metrics.ERROR_RATE: matrix_row.append({ FALSE_COUNT: false_count, COUNT: total_count, METRIC_NAME: metric_to_display_name[metric] }) elif is_multi_agg: matrix_row.append({ METRIC_VALUE: metric_value, TP: tp_sum, FP: fp_sum, FN: fn_sum, TN: tn_sum, ERROR: error, METRIC_NAME: metric_to_display_name[metric], COUNT: total_count }) else: matrix_row.append({ METRIC_VALUE: metric_value, METRIC_NAME: metric_to_display_name[metric], COUNT: total_count }) matrix.append(matrix_row) category2 = [] category2_min_interval = [] category2_max_interval = [] for cat2 in categories2: if isinstance(categories2, pd.IntervalIndex): category2_min_interval.append(cat2.left) category2_max_interval.append(cat2.right) category2.append(str(cat2)) else: category2.append(get_py_value(cat2)) category1 = { VALUES: category1, INTERVAL_MIN: category1_min_interval, INTERVAL_MAX: category1_max_interval } category2 = { VALUES: category2, INTERVAL_MIN: category2_min_interval, INTERVAL_MAX: category2_max_interval } return {MATRIX: matrix, CATEGORY1: category1, CATEGORY2: category2}
def validate_matrix_metric(matrix, exp_total_count, exp_total_error, metric, num_cat1, num_cat2): is_precision = metric in precision_metrics is_recall = metric in recall_metrics is_accuracy = metric == Metrics.ACCURACY_SCORE is_f1_score = metric in f1_metrics is_multi_agg = is_multi_agg_metric(metric) if metric == Metrics.ERROR_RATE: # take sum of count, false count total_count = 0 total_false_count = 0 for i in range(num_cat1): for j in range(num_cat2): cell_count = matrix[MATRIX][i][j][COUNT] assert cell_count >= 0 total_count += cell_count cell_false_count = matrix[MATRIX][i][j][FALSE_COUNT] assert cell_false_count >= 0 total_false_count += cell_false_count assert exp_total_count == total_count assert exp_total_error == total_false_count elif (metric == Metrics.MEAN_SQUARED_ERROR or metric == Metrics.MEAN_ABSOLUTE_ERROR): # take sum of count, metric value total_count = 0 total_metric_value = 0 for i in range(num_cat1): for j in range(num_cat2): count = matrix[MATRIX][i][j][COUNT] assert count >= 0 total_count += count cell_metric_value = matrix[MATRIX][i][j][METRIC_VALUE] assert cell_metric_value >= 0 cell_value = cell_metric_value * count total_metric_value += cell_value metric_name = matrix[MATRIX][i][j][METRIC_NAME] assert metric_name == metric_to_display_name[metric] total_metric_value = total_metric_value / total_count assert exp_total_count == total_count assert abs(exp_total_error - total_metric_value) < TOLERANCE elif is_multi_agg: # compute the overall metric from the data in each of the cells total_count = 0 total_metric_value = 0 cell_tp_value = None cell_tn_value = None cell_fp_value = None cell_fn_value = None for i in range(num_cat1): for j in range(num_cat2): count = matrix[MATRIX][i][j][COUNT] assert count >= 0 total_count += count cell_metric_value = matrix[MATRIX][i][j][METRIC_VALUE] assert cell_metric_value >= 0 if cell_tp_value is None: cell_tp_value = np.array(matrix[MATRIX][i][j][TP], dtype=FLOAT64) else: cell_tp_value += np.array(matrix[MATRIX][i][j][TP], dtype=FLOAT64) if is_precision or is_accuracy or is_f1_score: if cell_fp_value is None: cell_fp_value = np.array(matrix[MATRIX][i][j][FP], dtype=FLOAT64) else: cell_fp_value += np.array(matrix[MATRIX][i][j][FP], dtype=FLOAT64) if is_recall or is_accuracy or is_f1_score: if cell_fn_value is None: cell_fn_value = np.array(matrix[MATRIX][i][j][FN], dtype=FLOAT64) else: cell_fn_value += np.array(matrix[MATRIX][i][j][FN], dtype=FLOAT64) if is_accuracy: if cell_tn_value is None: cell_tn_value = np.array(matrix[MATRIX][i][j][TN], dtype=FLOAT64) else: cell_tn_value += np.array(matrix[MATRIX][i][j][TN], dtype=FLOAT64) metric_name = matrix[MATRIX][i][j][METRIC_NAME] assert metric_name == metric_to_display_name[metric] tp_sum = cell_tp_value.sum() is_overall_precision = (metric == Metrics.MICRO_PRECISION_SCORE or metric == Metrics.PRECISION_SCORE) is_overall_recall = (metric == Metrics.MICRO_RECALL_SCORE or metric == Metrics.RECALL_SCORE) is_overall_f1_score = (metric == Metrics.MICRO_F1_SCORE or metric == Metrics.F1_SCORE) if is_overall_precision: total_metric_value = tp_sum / (tp_sum + cell_fp_value.sum()) elif is_overall_recall: total_metric_value = tp_sum / (tp_sum + cell_fn_value.sum()) elif metric == Metrics.MACRO_PRECISION_SCORE: per_class_metrics = cell_tp_value / (cell_tp_value + cell_fp_value) num_classes = len(per_class_metrics) total_metric_value = per_class_metrics.sum() / num_classes elif metric == Metrics.MACRO_RECALL_SCORE: per_class_metrics = cell_tp_value / (cell_tp_value + cell_fn_value) num_classes = len(per_class_metrics) total_metric_value = per_class_metrics.sum() / num_classes elif metric == Metrics.PRECISION_SCORE: total_metric_value = tp_sum / (tp_sum + cell_fp_value.sum()) elif metric == Metrics.ACCURACY_SCORE: if len(cell_tn_value) < 2: tn_sum = cell_tn_value.sum() fn_sum = cell_fn_value.sum() fp_sum = cell_fp_value.sum() num_correct = tp_sum + tn_sum num_total = num_correct + fn_sum + fp_sum total_metric_value = num_correct / num_total else: num_total = (cell_tp_value[0] + cell_tn_value[0] + cell_fn_value[0] + cell_fp_value[0]) total_metric_value = tp_sum / num_total elif is_overall_f1_score: fn_sum = cell_fn_value.sum() fp_sum = cell_fp_value.sum() total_metric_value = tp_sum / (tp_sum + (fp_sum + fn_sum) / 2) elif metric == Metrics.MACRO_F1_SCORE: pc_precision = cell_tp_value / (cell_tp_value + cell_fp_value) pc_recall = cell_tp_value / (cell_tp_value + cell_fn_value) num_classes = len(pc_precision) pc_f1_score = (2 * (pc_precision * pc_recall) / (pc_precision + pc_recall)).sum() total_metric_value = pc_f1_score / num_classes assert exp_total_count == total_count assert abs(exp_total_error - total_metric_value) < TOLERANCE else: raise NotImplementedError( "Metric {} validation not supported yet".format(metric))
def matrix_1d(categories, values_err, counts, counts_err, metric): """Constructs a 1D matrix. The matrix is in a dictionary format which can then be saved to JSON. :param categories: The categories for the selected feature. :type categories: list :param values_err: The error values for the selected feature. :type values_err: list :param counts: The counts for the selected feature. :type counts: list :param counts_err: The error counts for the selected feature. :type counts_err: list :param metric: The calculated metric. :type metric: str :returns: The 1D matrix dictionary. :rtype: dict """ matrix = [] matrix_row = [] for col_idx in reversed(range(len(categories))): cat = categories[col_idx] if metric == Metrics.ERROR_RATE: false_count = 0 if cat in values_err: index_err = list(values_err).index(cat) false_count = int(counts_err[index_err]) matrix_row.append({ FALSE_COUNT: false_count, COUNT: int(counts[col_idx]), METRIC_NAME: metric_to_display_name[metric] }) elif is_multi_agg_metric(metric): tp_sum = [] fp_sum = [] fn_sum = [] tn_sum = [] metric_value = 0 error = 0 if cat in values_err: metric_value = float(counts_err[col_idx][0]) tp_sum = counts_err[col_idx][1] fp_sum = counts_err[col_idx][2] fn_sum = counts_err[col_idx][3] tn_sum = counts_err[col_idx][4] error = float(counts_err[col_idx][5]) if math.isnan(metric_value): metric_value = 0.0 matrix_row.append({ METRIC_VALUE: metric_value, TP: tp_sum, FP: fp_sum, FN: fn_sum, TN: tn_sum, ERROR: error, METRIC_NAME: metric_to_display_name[metric], COUNT: int(counts[col_idx]) }) else: metric_value = 0 if cat in values_err: metric_value = float(counts_err[col_idx]) if math.isnan(metric_value): metric_value = 0.0 matrix_row.append({ METRIC_VALUE: metric_value, METRIC_NAME: metric_to_display_name[metric], COUNT: int(counts[col_idx]) }) matrix.append(matrix_row) category = [] category_min_interval = [] category_max_interval = [] for cat in reversed(categories): if isinstance(categories, pd.IntervalIndex): category_min_interval.append(cat.left) category_max_interval.append(cat.right) category.append(str(cat)) else: category.append(get_py_value(cat)) category1 = { VALUES: category, INTERVAL_MIN: category_min_interval, INTERVAL_MAX: category_max_interval } return {MATRIX: matrix, CATEGORY1: category1}
def compute_matrix(analyzer, features, filters, composite_filters, quantile_binning=False, num_bins=BIN_THRESHOLD): """Compute a matrix of metrics for a given set of feature names. The filters and composite filters are used to filter the data prior to computing the matrix. :param analyzer: The error analyzer. :type analyzer: BaseAnalyzer :param features: A list of one or two feature names to compute metrics for. :type features: list :param filters: A list of filters to apply to the data. :type filters: list :param composite_filters: A list of composite filters to apply to the data. :type composite_filters: list :param quantile_binning: Whether to use quantile binning. :type quantile_binning: bool :param num_bins: The number of bins to use for quantile binning. :type num_bins: int :return: A dictionary representation of the computed matrix which can be saved to JSON. :rtype: dict :Example: An example of running compute_matrix with a filter and a composite filter: >>> from erroranalysis._internal.error_analyzer import ModelAnalyzer >>> from erroranalysis._internal.matrix_filter import ( ... compute_matrix) >>> from erroranalysis._internal.constants import ModelTask >>> from sklearn.datasets import load_breast_cancer >>> from sklearn.model_selection import train_test_split >>> from sklearn import svm >>> breast_cancer_data = load_breast_cancer() >>> feature_names = breast_cancer_data.feature_names >>> X_train, X_test, y_train, y_test = train_test_split( ... breast_cancer_data.data, breast_cancer_data.target, ... test_size=0.5, random_state=0) >>> categorical_features = [] >>> clf = svm.SVC(gamma=0.001, C=100., probability=True, ... random_state=777) >>> model = clf.fit(X_train, y_train) >>> model_task = ModelTask.CLASSIFICATION >>> analyzer = ModelAnalyzer(model, X_test, y_test, feature_names, ... categorical_features, model_task=model_task) >>> filters = [{'arg': [23.85], 'column': 'mean radius', ... 'method': 'less and equal'}] >>> composite_filters = [{'compositeFilters': ... [{'compositeFilters': ... [{'arg': [13.45, 22.27], ... 'column': 'mean radius', ... 'method': 'in the range of'}, ... {'arg': [10.88, 24.46], ... 'column': 'mean texture', ... 'method': 'in the range of'}], ... 'operation': 'and'}], ... 'operation': 'or'}] >>> matrix = compute_matrix(analyzer, ['mean radius', 'mean texture'], ... filters, composite_filters) """ if num_bins <= 0: raise ValueError( 'Number of bins parameter must be greater than 0 for the heatmap') if features[0] is None and features[1] is None: raise ValueError( 'One or two features must be specified to compute the heat map') filtered_df = filter_from_cohort(analyzer, filters, composite_filters) true_y = filtered_df[TRUE_Y] dropped_cols = [TRUE_Y, ROW_INDEX] is_model_analyzer = hasattr(analyzer, 'model') if not is_model_analyzer: pred_y = filtered_df[PRED_Y] dropped_cols.append(PRED_Y) input_data = filtered_df.drop(columns=dropped_cols) is_pandas = isinstance(analyzer.dataset, pd.DataFrame) metric = analyzer.metric if is_pandas: true_y = true_y.to_numpy() else: input_data = input_data.to_numpy() if is_model_analyzer: pred_y = analyzer.model.predict(input_data) if is_model_analyzer: if analyzer.model_task == ModelTask.CLASSIFICATION: diff = analyzer.model.predict(input_data) != true_y else: diff = analyzer.model.predict(input_data) - true_y else: if analyzer.model_task == ModelTask.CLASSIFICATION: diff = pred_y != true_y else: diff = pred_y - true_y if not isinstance(diff, np.ndarray): diff = np.array(diff) if not isinstance(pred_y, np.ndarray): pred_y = np.array(pred_y) if not isinstance(true_y, np.ndarray): true_y = np.array(true_y) indexes = [] for feature in features: if feature is None: continue indexes.append(analyzer.feature_names.index(feature)) if is_pandas: input_data = input_data.to_numpy() dataset_sub_features = input_data[:, indexes] dataset_sub_names = np.array(analyzer.feature_names)[np.array(indexes)] df = pd.DataFrame(dataset_sub_features, columns=dataset_sub_names) df_err = df.copy() df_err[DIFF] = diff if metric == Metrics.ERROR_RATE: df_err = df_err[df_err[DIFF]] else: df_err[TRUE_Y] = true_y df_err[PRED_Y] = pred_y # construct matrix matrix = [] if len(dataset_sub_names) == 2: feat1 = dataset_sub_names[0] feat2 = dataset_sub_names[1] unique_count1 = len(df[feat1].unique()) unique_count2 = len(df[feat2].unique()) f1_is_cat = False f2_is_cat = False if analyzer.categorical_features is not None: f1_is_cat = feat1 in analyzer.categorical_features f2_is_cat = feat2 in analyzer.categorical_features if unique_count1 > num_bins and not f1_is_cat: tabdf1 = bin_data(df, feat1, num_bins, quantile_binning=quantile_binning) categories1 = tabdf1.cat.categories if len(categories1) < num_bins: warn_duplicate_edges(feat1) tabdf1_err = bin_data(df_err, feat1, categories1, quantile_binning=quantile_binning) else: tabdf1 = df[feat1] tabdf1_err = df_err[feat1] categories1 = np.unique(tabdf1.to_numpy(), return_counts=True)[0] if unique_count2 > num_bins and not f2_is_cat: tabdf2 = bin_data(df, feat2, num_bins, quantile_binning=quantile_binning) categories2 = tabdf2.cat.categories if len(categories2) < num_bins: warn_duplicate_edges(feat2) tabdf2_err = bin_data(df_err, feat2, categories2, quantile_binning=quantile_binning) else: tabdf2 = df[feat2] tabdf2_err = df_err[feat2] categories2 = np.unique(tabdf2.to_numpy(), return_counts=True)[0] if metric == Metrics.ERROR_RATE: matrix_total = pd.crosstab(tabdf1, tabdf2, rownames=[feat1], colnames=[feat2]) matrix_error = pd.crosstab(tabdf1_err, tabdf2_err, rownames=[feat1], colnames=[feat2]) else: if is_multi_agg_metric(metric): ordered_labels = get_ordered_classes(analyzer.classes, true_y, pred_y) aggfunc = _MultiMetricAggFunc(metric_to_func[metric], ordered_labels, metric) else: aggfunc = _AggFunc(metric_to_func[metric]) matrix_total = pd.crosstab(tabdf1, tabdf2, rownames=[feat1], colnames=[feat2]) matrix_error = pd.crosstab(tabdf1_err, tabdf2_err, rownames=[feat1], colnames=[feat2], values=list( zip(df_err[TRUE_Y], df_err[PRED_Y])), aggfunc=aggfunc._agg_func_pair) fill_matrix_nulls(matrix_total, aggfunc._fill_na_value()) fill_matrix_nulls(matrix_error, aggfunc._fill_na_value()) matrix = matrix_2d(categories1, categories2, matrix_total, matrix_error, metric) else: feat1 = dataset_sub_names[0] unique_count1 = len(df[feat1].unique()) f1_is_cat = False if analyzer.categorical_features is not None: f1_is_cat = feat1 in analyzer.categorical_features if unique_count1 > num_bins and not f1_is_cat: cutdf = bin_data(df, feat1, num_bins, quantile_binning=quantile_binning) num_categories = len(cutdf.cat.categories) bin_range = range(num_categories) if len(cutdf.cat.categories) < num_bins: warn_duplicate_edges(feat1) catr = cutdf.cat.rename_categories(bin_range) catn, counts = np.unique(catr.to_numpy(), return_counts=True) # fix counts to include skipped categories fix_counts = [] counts_idx = 0 for idx, catdf in enumerate(cutdf.cat.categories): if idx not in catn: fix_counts.append(0) else: fix_counts.append(counts[counts_idx]) counts_idx += 1 counts = fix_counts cut_err = bin_data(df_err, feat1, cutdf.cat.categories, quantile_binning=quantile_binning) catr_err = cut_err.cat.rename_categories(bin_range) val_err, counts_err = np.unique(catr_err.to_numpy(), return_counts=True) val_err = cut_err.cat.categories[val_err] categories = cutdf.cat.categories else: categories, counts = np.unique(df[feat1].to_numpy(), return_counts=True) val_err, counts_err = np.unique(df_err[feat1].to_numpy(), return_counts=True) cut_err = df_err # Compute the given metric for each group, if not using error rate if metric != Metrics.ERROR_RATE: if is_multi_agg_metric(metric): ordered_labels = get_ordered_classes(analyzer.classes, true_y, pred_y) aggfunc = _MultiMetricAggFunc(metric_to_func[metric], ordered_labels, metric) else: aggfunc = _AggFunc(metric_to_func[metric]) cutdf_err = pd.DataFrame(cut_err) cutdf_err['metric_values'] = list( zip(df_err[TRUE_Y], df_err[PRED_Y])) grouped = cutdf_err.groupby([feat1]) agg_func = {'metric_values': aggfunc._agg_func_grouped} counts_err = grouped.agg(agg_func) counts_err = counts_err.values.ravel() matrix = matrix_1d(categories, val_err, counts, counts_err, metric) return matrix
def matrix_2d(categories1, categories2, matrix_counts, matrix_err_counts, metric): """Constructs a 2D matrix. The matrix is in a dictionary format which can then be saved to JSON. :param categories1: The categories for the first selected feature. :type categories1: list :param categories2: The categories for the second selected feature. :type categories2: list :param matrix_counts: The matrix counts. :type matrix_counts: numpy.ndarray :param matrix_err_counts: The matrix error counts. :type matrix_err_counts: numpy.ndarray :param metric: The calculated metric. :type metric: str :returns: The 2D matrix dictionary. :rtype: dict """ matrix = [] category1 = [] category1_min_interval = [] category1_max_interval = [] is_multi_agg = is_multi_agg_metric(metric) for row_index in reversed(range(len(categories1))): matrix_row = [] cat1 = categories1[row_index] if isinstance(categories1, pd.IntervalIndex): category1_min_interval.append(cat1.left) category1_max_interval.append(cat1.right) category1.append(str(cat1)) else: category1.append(get_py_value(cat1)) for col_index in range(len(categories2)): cat2 = categories2[col_index] index_exists_err = cat1 in matrix_err_counts.index col_exists_err = cat2 in matrix_err_counts.columns if metric == Metrics.ERROR_RATE: false_count = 0 if index_exists_err and col_exists_err: false_count = int(matrix_err_counts.loc[cat1, cat2]) elif is_multi_agg: tp_sum = [] fp_sum = [] fn_sum = [] tn_sum = [] metric_value = 0 if index_exists_err and col_exists_err: metric_value = float(matrix_err_counts.loc[cat1, cat2][0]) tp_sum = matrix_err_counts.loc[cat1, cat2][1] fp_sum = matrix_err_counts.loc[cat1, cat2][2] fn_sum = matrix_err_counts.loc[cat1, cat2][3] tn_sum = matrix_err_counts.loc[cat1, cat2][4] error = float(matrix_err_counts.loc[cat1, cat2][5]) else: metric_value = 0 if index_exists_err and col_exists_err: metric_value = float(matrix_err_counts.loc[cat1, cat2]) index_exists = cat1 in matrix_counts.index col_exists = cat2 in matrix_counts.columns total_count = 0 if index_exists and col_exists: total_count = int(matrix_counts.loc[cat1, cat2]) if metric == Metrics.ERROR_RATE: matrix_row.append({ FALSE_COUNT: false_count, COUNT: total_count, METRIC_NAME: metric_to_display_name[metric] }) elif is_multi_agg: matrix_row.append({ METRIC_VALUE: metric_value, TP: tp_sum, FP: fp_sum, FN: fn_sum, TN: tn_sum, ERROR: error, METRIC_NAME: metric_to_display_name[metric], COUNT: total_count }) else: matrix_row.append({ METRIC_VALUE: metric_value, METRIC_NAME: metric_to_display_name[metric], COUNT: total_count }) matrix.append(matrix_row) category2 = [] category2_min_interval = [] category2_max_interval = [] for cat2 in categories2: if isinstance(categories2, pd.IntervalIndex): category2_min_interval.append(cat2.left) category2_max_interval.append(cat2.right) category2.append(str(cat2)) else: category2.append(get_py_value(cat2)) category1 = { VALUES: category1, INTERVAL_MIN: category1_min_interval, INTERVAL_MAX: category1_max_interval } category2 = { VALUES: category2, INTERVAL_MIN: category2_min_interval, INTERVAL_MAX: category2_max_interval } return {MATRIX: matrix, CATEGORY1: category1, CATEGORY2: category2}