def run_error_analyzer(validation_data,
model,
X_test,
y_test,
feature_names,
categorical_features,
model_task,
filters=None,
composite_filters=None,
is_empty_validation_data=False):
error_analyzer = ModelAnalyzer(model,
X_test,
y_test,
feature_names,
categorical_features,
model_task=model_task)
filtered_data = filter_from_cohort(error_analyzer, filters,
composite_filters)
# validate there is some data selected for each of the filters
if is_empty_validation_data:
assert validation_data.shape[0] == 0
else:
assert validation_data.shape[0] > 0
assert validation_data.equals(filtered_data)
def run_error_analyzer(model,
X_test,
y_test,
feature_names,
analyzer_type,
categorical_features=None,
tree_features=None,
max_depth=3,
num_leaves=31,
min_child_samples=20,
filters=None,
composite_filters=None,
metric=None,
model_task=None):
if analyzer_type == AnalyzerType.MODEL:
error_analyzer = ModelAnalyzer(model,
X_test,
y_test,
feature_names,
categorical_features,
metric=metric,
model_task=model_task)
else:
pred_y = model.predict(X_test)
error_analyzer = PredictionsAnalyzer(pred_y,
X_test,
y_test,
feature_names,
categorical_features,
metric=metric,
model_task=model_task)
if tree_features is None:
tree_features = feature_names
tree = error_analyzer.compute_error_tree(
tree_features,
filters,
composite_filters,
max_depth=max_depth,
num_leaves=num_leaves,
min_child_samples=min_child_samples)
validation_data = X_test
if filters is not None or composite_filters is not None:
validation_data = filter_from_cohort(error_analyzer, filters,
composite_filters)
y_test = validation_data[TRUE_Y]
validation_data = validation_data.drop(columns=[TRUE_Y, ROW_INDEX])
if not isinstance(X_test, pd.DataFrame):
validation_data = validation_data.values
validation_data_len = len(validation_data)
assert tree is not None
assert len(tree) > 0
assert ERROR in tree[0]
assert ID in tree[0]
assert PARENTID in tree[0]
assert tree[0][PARENTID] is None
assert SIZE in tree[0]
assert tree[0][SIZE] == validation_data_len
for node in tree:
assert node[SIZE] >= min(min_child_samples, validation_data_len)
def run_error_analyzer(model,
X_test,
y_test,
feature_names,
categorical_features,
model_task,
filters=None,
composite_filters=None,
matrix_features=None):
error_analyzer = ModelAnalyzer(model,
X_test,
y_test,
feature_names,
categorical_features,
model_task=model_task)
# features, filters, composite_filters
if matrix_features is None:
features = [feature_names[0], feature_names[1]]
else:
features = matrix_features
json_matrix = error_analyzer.compute_matrix(features, filters,
composite_filters)
validation_data = X_test
if filters is not None or composite_filters is not None:
validation_data = filter_from_cohort(X_test, filters,
composite_filters, feature_names,
y_test, categorical_features,
error_analyzer.categories)
y_test = validation_data[TRUE_Y]
validation_data = validation_data.drop(columns=[TRUE_Y, ROW_INDEX])
if not isinstance(X_test, pd.DataFrame):
validation_data = validation_data.values
expected_count = len(validation_data)
metric = error_analyzer.metric
if metric == Metrics.ERROR_RATE:
expected_error = sum(model.predict(validation_data) != y_test)
elif metric == Metrics.MEAN_SQUARED_ERROR:
func = metric_to_func[metric]
pred_y = model.predict(validation_data)
expected_error = func(y_test, pred_y)
else:
raise NotImplementedError(
"Metric {} validation not supported yet".format(metric))
validate_matrix(json_matrix,
expected_count,
expected_error,
features,
metric=metric)
def run_error_analyzer(model,
X_test,
y_test,
feature_names,
categorical_features,
model_task,
filters=None,
composite_filters=None,
matrix_features=None,
quantile_binning=False,
num_bins=BIN_THRESHOLD,
metric=None):
error_analyzer = ModelAnalyzer(model,
X_test,
y_test,
feature_names,
categorical_features,
model_task=model_task,
metric=metric)
# features, filters, composite_filters
if matrix_features is None:
features = [feature_names[0], feature_names[1]]
else:
features = matrix_features
matrix = error_analyzer.compute_matrix(features,
filters,
composite_filters,
quantile_binning=quantile_binning,
num_bins=num_bins)
validation_data = X_test
if filters is not None or composite_filters is not None:
validation_data = filter_from_cohort(error_analyzer, filters,
composite_filters)
y_test = validation_data[TRUE_Y]
validation_data = validation_data.drop(columns=[TRUE_Y, ROW_INDEX])
if not isinstance(X_test, pd.DataFrame):
validation_data = validation_data.values
expected_count = len(validation_data)
metric = error_analyzer.metric
expected_error = get_expected_metric_error(error_analyzer, metric, model,
validation_data, y_test)
validate_matrix(matrix,
expected_count,
expected_error,
features,
metric=metric)
def compute_json_error_tree(analyzer, features, filters, composite_filters):
# Fit a surrogate model on errors
surrogate = LGBMClassifier(n_estimators=1, max_depth=3)
is_model_analyzer = hasattr(analyzer, MODEL)
if is_model_analyzer:
filtered_df = filter_from_cohort(analyzer.dataset, filters,
composite_filters,
analyzer.feature_names,
analyzer.true_y,
analyzer.categorical_features,
analyzer.categories)
else:
filtered_df = filter_from_cohort(analyzer.dataset, filters,
composite_filters,
analyzer.feature_names,
analyzer.true_y,
analyzer.categorical_features,
analyzer.categories, analyzer.pred_y)
row_index = filtered_df[ROW_INDEX]
true_y = filtered_df[TRUE_Y]
dropped_cols = [TRUE_Y, ROW_INDEX]
if not is_model_analyzer:
pred_y = filtered_df[PRED_Y]
dropped_cols.append(PRED_Y)
input_data = filtered_df.drop(columns=dropped_cols)
is_pandas = isinstance(analyzer.dataset, pd.DataFrame)
if is_pandas:
true_y = true_y.to_numpy()
else:
input_data = input_data.to_numpy()
if is_model_analyzer:
diff = analyzer.model.predict(input_data) != true_y
else:
diff = pred_y != true_y
if not isinstance(diff, np.ndarray):
diff = np.array(diff)
indexes = []
for feature in features:
indexes.append(analyzer.feature_names.index(feature))
if is_pandas:
input_data = input_data.to_numpy()
cat_ind_reindexed = []
categories_reindexed = []
if analyzer.categorical_features:
# Inplace replacement of columns
for idx, c_i in enumerate(analyzer.categorical_indexes):
input_data[:, c_i] = analyzer.string_indexed_data[row_index, idx]
dataset_sub_features = input_data[:, indexes]
dataset_sub_names = np.array(analyzer.feature_names)[np.array(indexes)]
dataset_sub_names = list(dataset_sub_names)
if analyzer.categorical_features:
for c_index, feature in enumerate(analyzer.categorical_features):
try:
index_sub = dataset_sub_names.index(feature)
except ValueError:
continue
cat_ind_reindexed.append(index_sub)
categories_reindexed.append(analyzer.categories[c_index])
surrogate.fit(dataset_sub_features,
diff,
categorical_feature=cat_ind_reindexed)
else:
surrogate.fit(dataset_sub_features, diff)
filtered_indexed_df = pd.DataFrame(dataset_sub_features,
columns=dataset_sub_names)
filtered_indexed_df[DIFF] = diff
model_json = surrogate._Booster.dump_model()
tree_structure = model_json["tree_info"][0]['tree_structure']
max_split_index = get_max_split_index(tree_structure) + 1
json_tree = traverse(filtered_indexed_df, tree_structure, max_split_index,
(categories_reindexed, cat_ind_reindexed), [],
dataset_sub_names)
return json_tree
def compute_json_matrix(analyzer, features, filters, composite_filters):
if features[0] is None and features[1] is None:
raise ValueError(
"One or two features must be specified to compute the heat map")
is_model_analyzer = hasattr(analyzer, 'model')
if is_model_analyzer:
filtered_df = filter_from_cohort(analyzer.dataset, filters,
composite_filters,
analyzer.feature_names,
analyzer.true_y,
analyzer.categorical_features,
analyzer.categories)
else:
filtered_df = filter_from_cohort(analyzer.dataset, filters,
composite_filters,
analyzer.feature_names,
analyzer.true_y,
analyzer.categorical_features,
analyzer.categories, analyzer.pred_y)
true_y = filtered_df[TRUE_Y]
dropped_cols = [TRUE_Y, ROW_INDEX]
if not is_model_analyzer:
pred_y = filtered_df[PRED_Y]
dropped_cols.append(PRED_Y)
input_data = filtered_df.drop(columns=dropped_cols)
is_pandas = isinstance(analyzer.dataset, pd.DataFrame)
metric = analyzer.metric
if is_pandas:
true_y = true_y.to_numpy()
else:
input_data = input_data.to_numpy()
if is_model_analyzer:
pred_y = analyzer.model.predict(input_data)
if is_model_analyzer:
if analyzer.model_task == ModelTask.CLASSIFICATION:
diff = analyzer.model.predict(input_data) != true_y
else:
diff = analyzer.model.predict(input_data) - true_y
else:
if analyzer.model_task == ModelTask.CLASSIFICATION:
diff = pred_y != true_y
else:
diff = pred_y - true_y
if not isinstance(diff, np.ndarray):
diff = np.array(diff)
if not isinstance(pred_y, np.ndarray):
pred_y = np.array(pred_y)
if not isinstance(true_y, np.ndarray):
true_y = np.array(true_y)
indexes = []
for feature in features:
if feature is None:
continue
indexes.append(analyzer.feature_names.index(feature))
if is_pandas:
input_data = input_data.to_numpy()
dataset_sub_features = input_data[:, indexes]
dataset_sub_names = np.array(analyzer.feature_names)[np.array(indexes)]
df = pd.DataFrame(dataset_sub_features, columns=dataset_sub_names)
df_err = df.copy()
df_err[DIFF] = diff
if analyzer.model_task == ModelTask.CLASSIFICATION:
df_err = df_err[df_err[DIFF]]
else:
df_err[TRUE_Y] = true_y
df_err[PRED_Y] = pred_y
# construct json matrix
json_matrix = []
if len(dataset_sub_names) == 2:
feat1 = dataset_sub_names[0]
feat2 = dataset_sub_names[1]
unique_count1 = len(df[feat1].unique())
unique_count2 = len(df[feat2].unique())
f1_is_cat = False
f2_is_cat = False
if analyzer.categorical_features is not None:
f1_is_cat = feat1 in analyzer.categorical_features
f2_is_cat = feat2 in analyzer.categorical_features
if unique_count1 > BIN_THRESHOLD and not f1_is_cat:
tabdf1, bins = pd.cut(df[feat1], BIN_THRESHOLD, retbins=True)
tabdf1_err = pd.cut(df_err[feat1], bins)
categories1 = tabdf1.cat.categories
else:
tabdf1 = df[feat1]
tabdf1_err = df_err[feat1]
categories1 = np.unique(tabdf1.to_numpy(), return_counts=True)[0]
if unique_count2 > BIN_THRESHOLD and not f2_is_cat:
tabdf2, bins = pd.cut(df[feat2], BIN_THRESHOLD, retbins=True)
tabdf2_err = pd.cut(df_err[feat2], bins)
categories2 = tabdf2.cat.categories
else:
tabdf2 = df[feat2]
tabdf2_err = df_err[feat2]
categories2 = np.unique(tabdf2.to_numpy(), return_counts=True)[0]
if metric == Metrics.ERROR_RATE:
matrix_total = pd.crosstab(tabdf1,
tabdf2,
rownames=[feat1],
colnames=[feat2])
matrix_error = pd.crosstab(tabdf1_err,
tabdf2_err,
rownames=[feat1],
colnames=[feat2])
else:
aggfunc = _AggFunc(metric_to_func[metric])
matrix_total = pd.crosstab(tabdf1,
tabdf2,
rownames=[feat1],
colnames=[feat2])
matrix_error = pd.crosstab(tabdf1_err,
tabdf2_err,
rownames=[feat1],
colnames=[feat2],
values=list(
zip(df_err[TRUE_Y],
df_err[PRED_Y])),
aggfunc=aggfunc._agg_func_pair)
matrix_total = matrix_total.fillna(0)
matrix_error = matrix_error.fillna(0)
json_matrix = json_matrix_2d(categories1, categories2, matrix_total,
matrix_error, metric)
else:
feat1 = dataset_sub_names[0]
unique_count1 = len(df[feat1].unique())
f1_is_cat = False
if analyzer.categorical_features is not None:
f1_is_cat = feat1 in analyzer.categorical_features
if unique_count1 > BIN_THRESHOLD and not f1_is_cat:
cutdf, bins = pd.cut(df[feat1], BIN_THRESHOLD, retbins=True)
bin_range = range(BIN_THRESHOLD)
catr = cutdf.cat.rename_categories(bin_range)
catn, counts = np.unique(catr.to_numpy(), return_counts=True)
# fix counts to include skipped categories
fix_counts = []
counts_idx = 0
for idx, catdf in enumerate(cutdf.cat.categories):
if idx not in catn:
fix_counts.append(0)
else:
fix_counts.append(counts[counts_idx])
counts_idx += 1
counts = fix_counts
cut_err = pd.cut(df_err[feat1], bins)
catr_err = cut_err.cat.rename_categories(bin_range)
val_err, counts_err = np.unique(catr_err.to_numpy(),
return_counts=True)
val_err = cut_err.cat.categories[val_err]
categories = cutdf.cat.categories
else:
categories, counts = np.unique(df[feat1].to_numpy(),
return_counts=True)
val_err, counts_err = np.unique(df_err[feat1].to_numpy(),
return_counts=True)
cut_err = df_err
# Compute the given metric for each group, if not using error rate
if metric != Metrics.ERROR_RATE:
aggfunc = _AggFunc(metric_to_func[metric])
cutdf_err = pd.DataFrame(cut_err)
cutdf_err['metric_values'] = list(
zip(df_err[TRUE_Y], df_err[PRED_Y]))
grouped = cutdf_err.groupby([feat1])
counts_err = grouped.agg(aggfunc._agg_func_triplet)
counts_err = counts_err.values.ravel()
json_matrix = json_matrix_1d(categories, val_err, counts, counts_err,
metric)
return json_matrix
def compute_error_tree(analyzer,
features,
filters,
composite_filters,
max_depth=DEFAULT_MAX_DEPTH,
num_leaves=DEFAULT_NUM_LEAVES,
min_child_samples=DEFAULT_MIN_CHILD_SAMPLES):
"""Computes the error tree for the given dataset.
:param analyzer: The error analyzer containing the categorical
features and categories for the full dataset.
:type analyzer: BaseAnalyzer
:param features: The features to train the surrogate model on.
:type features: numpy.ndarray or pandas.DataFrame
:param filters: The filters to apply to the dataset.
:type filters: numpy.ndarray or pandas.DataFrame
:param composite_filters: The composite filters to apply to the dataset.
:type composite_filters: numpy.ndarray or pandas.DataFrame
:param max_depth: The maximum depth of the surrogate tree trained
on errors.
:type max_depth: int
:param num_leaves: The number of leaves of the surrogate tree
trained on errors.
:type num_leaves: int
:param min_child_samples: The minimal number of data required to
create one leaf.
:return: The tree representation as a list of nodes.
:rtype: list[dict[str, str]]
:Example:
An example of running compute_error_tree with a
filter and a composite filter:
>>> from erroranalysis._internal.error_analyzer import ModelAnalyzer
>>> from erroranalysis._internal.surrogate_error_tree import (
... compute_error_tree)
>>> from erroranalysis._internal.constants import ModelTask
>>> from sklearn.datasets import load_breast_cancer
>>> from sklearn.model_selection import train_test_split
>>> from sklearn import svm
>>> breast_cancer_data = load_breast_cancer()
>>> feature_names = breast_cancer_data.feature_names
>>> X_train, X_test, y_train, y_test = train_test_split(
... breast_cancer_data.data, breast_cancer_data.target,
... test_size=0.2, random_state=0)
>>> categorical_features = []
>>> clf = svm.SVC(gamma=0.001, C=100., probability=True,
... random_state=777)
>>> model = clf.fit(X_train, y_train)
>>> model_task = ModelTask.CLASSIFICATION
>>> analyzer = ModelAnalyzer(model, X_test, y_test, feature_names,
... categorical_features, model_task=model_task)
>>> filters = [{'arg': [23.85], 'column': 'mean radius',
... 'method': 'less and equal'}]
>>> composite_filters = [{'compositeFilters':
... [{'compositeFilters':
... [{'arg': [13.45, 22.27],
... 'column': 'mean radius',
... 'method': 'in the range of'},
... {'arg': [10.88, 24.46],
... 'column': 'mean texture',
... 'method': 'in the range of'}],
... 'operation': 'and'}],
... 'operation': 'or'}]
>>> tree = compute_error_tree(analyzer, ['mean radius', 'mean texture'],
... filters, composite_filters)
"""
# Fit a surrogate model on errors
if max_depth is None:
max_depth = DEFAULT_MAX_DEPTH
if num_leaves is None:
num_leaves = DEFAULT_NUM_LEAVES
if min_child_samples is None:
min_child_samples = DEFAULT_MIN_CHILD_SAMPLES
filtered_df = filter_from_cohort(analyzer,
filters,
composite_filters)
if filtered_df.shape[0] == 0:
return create_empty_node(analyzer.metric)
is_model_analyzer = hasattr(analyzer, MODEL)
indexes = []
for feature in features:
indexes.append(analyzer.feature_names.index(feature))
dataset_sub_names = np.array(analyzer.feature_names)[np.array(indexes)]
dataset_sub_names = list(dataset_sub_names)
if not is_spark(filtered_df):
booster, filtered_indexed_df, cat_info = get_surrogate_booster_local(
filtered_df, analyzer, is_model_analyzer, indexes,
dataset_sub_names, max_depth, num_leaves, min_child_samples)
cat_ind_reindexed, categories_reindexed = cat_info
else:
booster, filtered_indexed_df = get_surrogate_booster_pyspark(
filtered_df, analyzer, max_depth, num_leaves, min_child_samples)
cat_ind_reindexed = []
categories_reindexed = []
dumped_model = booster.dump_model()
tree_structure = dumped_model["tree_info"][0]['tree_structure']
max_split_index = get_max_split_index(tree_structure) + 1
cache_subtree_features(tree_structure, dataset_sub_names)
tree = traverse(filtered_indexed_df,
tree_structure,
max_split_index,
(categories_reindexed,
cat_ind_reindexed),
[],
dataset_sub_names,
metric=analyzer.metric,
classes=analyzer.classes)
return tree
def compute_matrix(analyzer,
features,
filters,
composite_filters,
quantile_binning=False,
num_bins=BIN_THRESHOLD):
if num_bins <= 0:
raise ValueError(
'Number of bins parameter must be greater than 0 for the heatmap')
if features[0] is None and features[1] is None:
raise ValueError(
'One or two features must be specified to compute the heat map')
filtered_df = filter_from_cohort(analyzer, filters, composite_filters)
true_y = filtered_df[TRUE_Y]
dropped_cols = [TRUE_Y, ROW_INDEX]
is_model_analyzer = hasattr(analyzer, 'model')
if not is_model_analyzer:
pred_y = filtered_df[PRED_Y]
dropped_cols.append(PRED_Y)
input_data = filtered_df.drop(columns=dropped_cols)
is_pandas = isinstance(analyzer.dataset, pd.DataFrame)
metric = analyzer.metric
if is_pandas:
true_y = true_y.to_numpy()
else:
input_data = input_data.to_numpy()
if is_model_analyzer:
pred_y = analyzer.model.predict(input_data)
if is_model_analyzer:
if analyzer.model_task == ModelTask.CLASSIFICATION:
diff = analyzer.model.predict(input_data) != true_y
else:
diff = analyzer.model.predict(input_data) - true_y
else:
if analyzer.model_task == ModelTask.CLASSIFICATION:
diff = pred_y != true_y
else:
diff = pred_y - true_y
if not isinstance(diff, np.ndarray):
diff = np.array(diff)
if not isinstance(pred_y, np.ndarray):
pred_y = np.array(pred_y)
if not isinstance(true_y, np.ndarray):
true_y = np.array(true_y)
indexes = []
for feature in features:
if feature is None:
continue
indexes.append(analyzer.feature_names.index(feature))
if is_pandas:
input_data = input_data.to_numpy()
dataset_sub_features = input_data[:, indexes]
dataset_sub_names = np.array(analyzer.feature_names)[np.array(indexes)]
df = pd.DataFrame(dataset_sub_features, columns=dataset_sub_names)
df_err = df.copy()
df_err[DIFF] = diff
if metric == Metrics.ERROR_RATE:
df_err = df_err[df_err[DIFF]]
else:
df_err[TRUE_Y] = true_y
df_err[PRED_Y] = pred_y
# construct matrix
matrix = []
if len(dataset_sub_names) == 2:
feat1 = dataset_sub_names[0]
feat2 = dataset_sub_names[1]
unique_count1 = len(df[feat1].unique())
unique_count2 = len(df[feat2].unique())
f1_is_cat = False
f2_is_cat = False
if analyzer.categorical_features is not None:
f1_is_cat = feat1 in analyzer.categorical_features
f2_is_cat = feat2 in analyzer.categorical_features
if unique_count1 > num_bins and not f1_is_cat:
tabdf1 = bin_data(df,
feat1,
num_bins,
quantile_binning=quantile_binning)
categories1 = tabdf1.cat.categories
if len(categories1) < num_bins:
warn_duplicate_edges(feat1)
tabdf1_err = bin_data(df_err,
feat1,
categories1,
quantile_binning=quantile_binning)
else:
tabdf1 = df[feat1]
tabdf1_err = df_err[feat1]
categories1 = np.unique(tabdf1.to_numpy(), return_counts=True)[0]
if unique_count2 > num_bins and not f2_is_cat:
tabdf2 = bin_data(df,
feat2,
num_bins,
quantile_binning=quantile_binning)
categories2 = tabdf2.cat.categories
if len(categories2) < num_bins:
warn_duplicate_edges(feat2)
tabdf2_err = bin_data(df_err,
feat2,
categories2,
quantile_binning=quantile_binning)
else:
tabdf2 = df[feat2]
tabdf2_err = df_err[feat2]
categories2 = np.unique(tabdf2.to_numpy(), return_counts=True)[0]
if metric == Metrics.ERROR_RATE:
matrix_total = pd.crosstab(tabdf1,
tabdf2,
rownames=[feat1],
colnames=[feat2])
matrix_error = pd.crosstab(tabdf1_err,
tabdf2_err,
rownames=[feat1],
colnames=[feat2])
else:
if is_multi_agg_metric(metric):
ordered_labels = get_ordered_classes(analyzer.classes, true_y,
pred_y)
aggfunc = _MultiMetricAggFunc(metric_to_func[metric],
ordered_labels, metric)
else:
aggfunc = _AggFunc(metric_to_func[metric])
matrix_total = pd.crosstab(tabdf1,
tabdf2,
rownames=[feat1],
colnames=[feat2])
matrix_error = pd.crosstab(tabdf1_err,
tabdf2_err,
rownames=[feat1],
colnames=[feat2],
values=list(
zip(df_err[TRUE_Y],
df_err[PRED_Y])),
aggfunc=aggfunc._agg_func_pair)
fill_matrix_nulls(matrix_total, aggfunc._fill_na_value())
fill_matrix_nulls(matrix_error, aggfunc._fill_na_value())
matrix = matrix_2d(categories1, categories2, matrix_total,
matrix_error, metric)
else:
feat1 = dataset_sub_names[0]
unique_count1 = len(df[feat1].unique())
f1_is_cat = False
if analyzer.categorical_features is not None:
f1_is_cat = feat1 in analyzer.categorical_features
if unique_count1 > num_bins and not f1_is_cat:
cutdf = bin_data(df,
feat1,
num_bins,
quantile_binning=quantile_binning)
num_categories = len(cutdf.cat.categories)
bin_range = range(num_categories)
if len(cutdf.cat.categories) < num_bins:
warn_duplicate_edges(feat1)
catr = cutdf.cat.rename_categories(bin_range)
catn, counts = np.unique(catr.to_numpy(), return_counts=True)
# fix counts to include skipped categories
fix_counts = []
counts_idx = 0
for idx, catdf in enumerate(cutdf.cat.categories):
if idx not in catn:
fix_counts.append(0)
else:
fix_counts.append(counts[counts_idx])
counts_idx += 1
counts = fix_counts
cut_err = bin_data(df_err,
feat1,
cutdf.cat.categories,
quantile_binning=quantile_binning)
catr_err = cut_err.cat.rename_categories(bin_range)
val_err, counts_err = np.unique(catr_err.to_numpy(),
return_counts=True)
val_err = cut_err.cat.categories[val_err]
categories = cutdf.cat.categories
else:
categories, counts = np.unique(df[feat1].to_numpy(),
return_counts=True)
val_err, counts_err = np.unique(df_err[feat1].to_numpy(),
return_counts=True)
cut_err = df_err
# Compute the given metric for each group, if not using error rate
if metric != Metrics.ERROR_RATE:
if is_multi_agg_metric(metric):
ordered_labels = get_ordered_classes(analyzer.classes, true_y,
pred_y)
aggfunc = _MultiMetricAggFunc(metric_to_func[metric],
ordered_labels, metric)
else:
aggfunc = _AggFunc(metric_to_func[metric])
cutdf_err = pd.DataFrame(cut_err)
cutdf_err['metric_values'] = list(
zip(df_err[TRUE_Y], df_err[PRED_Y]))
grouped = cutdf_err.groupby([feat1])
agg_func = {'metric_values': aggfunc._agg_func_triplet}
counts_err = grouped.agg(agg_func)
counts_err = counts_err.values.ravel()
matrix = matrix_1d(categories, val_err, counts, counts_err, metric)
return matrix
def compute_json_matrix(analyzer, features, filters, composite_filters):
if features[0] is None and features[1] is None:
raise ValueError(
"One or two features must be specified to compute the heat map")
is_model_analyzer = hasattr(analyzer, 'model')
if is_model_analyzer:
filtered_df = filter_from_cohort(analyzer.dataset, filters,
composite_filters,
analyzer.feature_names,
analyzer.true_y,
analyzer.categorical_features,
analyzer.categories)
else:
filtered_df = filter_from_cohort(analyzer.dataset, filters,
composite_filters,
analyzer.feature_names,
analyzer.true_y,
analyzer.categorical_features,
analyzer.categories, analyzer.pred_y)
true_y = filtered_df[TRUE_Y]
dropped_cols = [TRUE_Y, ROW_INDEX]
if not is_model_analyzer:
pred_y = filtered_df[PRED_Y]
dropped_cols.append(PRED_Y)
input_data = filtered_df.drop(columns=dropped_cols)
is_pandas = isinstance(analyzer.dataset, pd.DataFrame)
if is_pandas:
true_y = true_y.to_numpy()
else:
input_data = input_data.to_numpy()
if is_model_analyzer:
diff = analyzer.model.predict(input_data) != true_y
else:
diff = pred_y != true_y
if not isinstance(diff, np.ndarray):
diff = np.array(diff)
indexes = []
for feature in features:
if feature is None:
continue
indexes.append(analyzer.feature_names.index(feature))
if is_pandas:
input_data = input_data.to_numpy()
dataset_sub_features = input_data[:, indexes]
dataset_sub_names = np.array(analyzer.feature_names)[np.array(indexes)]
df = pd.DataFrame(dataset_sub_features, columns=dataset_sub_names)
df_err = df.copy()
df_err[DIFF] = diff
df_err = df_err[df_err[DIFF]]
# construct json matrix
json_matrix = []
if len(dataset_sub_names) == 2:
feat1 = dataset_sub_names[0]
feat2 = dataset_sub_names[1]
unique_count1 = len(df[feat1].unique())
unique_count2 = len(df[feat2].unique())
f1_is_cat = False
f2_is_cat = False
if analyzer.categorical_features is not None:
f1_is_cat = feat1 in analyzer.categorical_features
f2_is_cat = feat2 in analyzer.categorical_features
if unique_count1 > BIN_THRESHOLD and not f1_is_cat:
tabdf1, bins = pd.cut(df[feat1], BIN_THRESHOLD, retbins=True)
tabdf1_err = pd.cut(df_err[feat1], bins)
categories1 = tabdf1.cat.categories
else:
tabdf1 = df[feat1]
tabdf1_err = df_err[feat1]
categories1 = np.unique(tabdf1.to_numpy(), return_counts=True)[0]
if unique_count2 > BIN_THRESHOLD and not f2_is_cat:
tabdf2, bins = pd.cut(df[feat2], BIN_THRESHOLD, retbins=True)
tabdf2_err = pd.cut(df_err[feat2], bins)
categories2 = tabdf2.cat.categories
else:
tabdf2 = df[feat2]
tabdf2_err = df_err[feat2]
categories2 = np.unique(tabdf2.to_numpy(), return_counts=True)[0]
matrix_total = pd.crosstab(tabdf1,
tabdf2,
rownames=[feat1],
colnames=[feat2])
matrix_error = pd.crosstab(tabdf1_err,
tabdf2_err,
rownames=[feat1],
colnames=[feat2])
json_matrix = json_matrix_2d(categories1, categories2, matrix_total,
matrix_error)
else:
feat1 = dataset_sub_names[0]
unique_count1 = len(df[feat1].unique())
f1_is_cat = False
if analyzer.categorical_features is not None:
f1_is_cat = feat1 in analyzer.categorical_features
if unique_count1 > BIN_THRESHOLD and not f1_is_cat:
cutdf, bins = pd.cut(df[feat1], BIN_THRESHOLD, retbins=True)
bin_range = range(BIN_THRESHOLD)
catr = cutdf.cat.rename_categories(bin_range)
catn, counts = np.unique(catr.to_numpy(), return_counts=True)
# fix counts to include skipped categories
fix_counts = []
counts_idx = 0
for idx, catdf in enumerate(cutdf.cat.categories):
if idx not in catn:
fix_counts.append(0)
else:
fix_counts.append(counts[counts_idx])
counts_idx += 1
counts = fix_counts
cutdf_err = pd.cut(df_err[feat1], bins)
catr_err = cutdf_err.cat.rename_categories(bin_range)
val_err, counts_err = np.unique(catr_err.to_numpy(),
return_counts=True)
val_err = cutdf_err.cat.categories[val_err]
json_matrix = json_matrix_1d(cutdf.cat.categories, val_err, counts,
counts_err)
else:
values, counts = np.unique(df[feat1].to_numpy(),
return_counts=True)
val_err, counts_err = np.unique(df_err[feat1].to_numpy(),
return_counts=True)
json_matrix = json_matrix_1d(values, val_err, counts, counts_err)
return json_matrix
def compute_matrix(analyzer,
features,
filters,
composite_filters,
quantile_binning=False,
num_bins=BIN_THRESHOLD):
"""Compute a matrix of metrics for a given set of feature names.
The filters and composite filters are used to filter the data
prior to computing the matrix.
:param analyzer: The error analyzer.
:type analyzer: BaseAnalyzer
:param features: A list of one or two feature names to compute metrics for.
:type features: list
:param filters: A list of filters to apply to the data.
:type filters: list
:param composite_filters: A list of composite filters to apply to the data.
:type composite_filters: list
:param quantile_binning: Whether to use quantile binning.
:type quantile_binning: bool
:param num_bins: The number of bins to use for quantile binning.
:type num_bins: int
:return: A dictionary representation of the computed matrix which can be
saved to JSON.
:rtype: dict
:Example:
An example of running compute_matrix with a filter and a composite
filter:
>>> from erroranalysis._internal.error_analyzer import ModelAnalyzer
>>> from erroranalysis._internal.matrix_filter import (
... compute_matrix)
>>> from erroranalysis._internal.constants import ModelTask
>>> from sklearn.datasets import load_breast_cancer
>>> from sklearn.model_selection import train_test_split
>>> from sklearn import svm
>>> breast_cancer_data = load_breast_cancer()
>>> feature_names = breast_cancer_data.feature_names
>>> X_train, X_test, y_train, y_test = train_test_split(
... breast_cancer_data.data, breast_cancer_data.target,
... test_size=0.5, random_state=0)
>>> categorical_features = []
>>> clf = svm.SVC(gamma=0.001, C=100., probability=True,
... random_state=777)
>>> model = clf.fit(X_train, y_train)
>>> model_task = ModelTask.CLASSIFICATION
>>> analyzer = ModelAnalyzer(model, X_test, y_test, feature_names,
... categorical_features, model_task=model_task)
>>> filters = [{'arg': [23.85], 'column': 'mean radius',
... 'method': 'less and equal'}]
>>> composite_filters = [{'compositeFilters':
... [{'compositeFilters':
... [{'arg': [13.45, 22.27],
... 'column': 'mean radius',
... 'method': 'in the range of'},
... {'arg': [10.88, 24.46],
... 'column': 'mean texture',
... 'method': 'in the range of'}],
... 'operation': 'and'}],
... 'operation': 'or'}]
>>> matrix = compute_matrix(analyzer, ['mean radius', 'mean texture'],
... filters, composite_filters)
"""
if num_bins <= 0:
raise ValueError(
'Number of bins parameter must be greater than 0 for the heatmap')
if features[0] is None and features[1] is None:
raise ValueError(
'One or two features must be specified to compute the heat map')
filtered_df = filter_from_cohort(analyzer, filters, composite_filters)
true_y = filtered_df[TRUE_Y]
dropped_cols = [TRUE_Y, ROW_INDEX]
is_model_analyzer = hasattr(analyzer, 'model')
if not is_model_analyzer:
pred_y = filtered_df[PRED_Y]
dropped_cols.append(PRED_Y)
input_data = filtered_df.drop(columns=dropped_cols)
is_pandas = isinstance(analyzer.dataset, pd.DataFrame)
metric = analyzer.metric
if is_pandas:
true_y = true_y.to_numpy()
else:
input_data = input_data.to_numpy()
if is_model_analyzer:
pred_y = analyzer.model.predict(input_data)
if is_model_analyzer:
if analyzer.model_task == ModelTask.CLASSIFICATION:
diff = analyzer.model.predict(input_data) != true_y
else:
diff = analyzer.model.predict(input_data) - true_y
else:
if analyzer.model_task == ModelTask.CLASSIFICATION:
diff = pred_y != true_y
else:
diff = pred_y - true_y
if not isinstance(diff, np.ndarray):
diff = np.array(diff)
if not isinstance(pred_y, np.ndarray):
pred_y = np.array(pred_y)
if not isinstance(true_y, np.ndarray):
true_y = np.array(true_y)
indexes = []
for feature in features:
if feature is None:
continue
indexes.append(analyzer.feature_names.index(feature))
if is_pandas:
input_data = input_data.to_numpy()
dataset_sub_features = input_data[:, indexes]
dataset_sub_names = np.array(analyzer.feature_names)[np.array(indexes)]
df = pd.DataFrame(dataset_sub_features, columns=dataset_sub_names)
df_err = df.copy()
df_err[DIFF] = diff
if metric == Metrics.ERROR_RATE:
df_err = df_err[df_err[DIFF]]
else:
df_err[TRUE_Y] = true_y
df_err[PRED_Y] = pred_y
# construct matrix
matrix = []
if len(dataset_sub_names) == 2:
feat1 = dataset_sub_names[0]
feat2 = dataset_sub_names[1]
unique_count1 = len(df[feat1].unique())
unique_count2 = len(df[feat2].unique())
f1_is_cat = False
f2_is_cat = False
if analyzer.categorical_features is not None:
f1_is_cat = feat1 in analyzer.categorical_features
f2_is_cat = feat2 in analyzer.categorical_features
if unique_count1 > num_bins and not f1_is_cat:
tabdf1 = bin_data(df,
feat1,
num_bins,
quantile_binning=quantile_binning)
categories1 = tabdf1.cat.categories
if len(categories1) < num_bins:
warn_duplicate_edges(feat1)
tabdf1_err = bin_data(df_err,
feat1,
categories1,
quantile_binning=quantile_binning)
else:
tabdf1 = df[feat1]
tabdf1_err = df_err[feat1]
categories1 = np.unique(tabdf1.to_numpy(), return_counts=True)[0]
if unique_count2 > num_bins and not f2_is_cat:
tabdf2 = bin_data(df,
feat2,
num_bins,
quantile_binning=quantile_binning)
categories2 = tabdf2.cat.categories
if len(categories2) < num_bins:
warn_duplicate_edges(feat2)
tabdf2_err = bin_data(df_err,
feat2,
categories2,
quantile_binning=quantile_binning)
else:
tabdf2 = df[feat2]
tabdf2_err = df_err[feat2]
categories2 = np.unique(tabdf2.to_numpy(), return_counts=True)[0]
if metric == Metrics.ERROR_RATE:
matrix_total = pd.crosstab(tabdf1,
tabdf2,
rownames=[feat1],
colnames=[feat2])
matrix_error = pd.crosstab(tabdf1_err,
tabdf2_err,
rownames=[feat1],
colnames=[feat2])
else:
if is_multi_agg_metric(metric):
ordered_labels = get_ordered_classes(analyzer.classes, true_y,
pred_y)
aggfunc = _MultiMetricAggFunc(metric_to_func[metric],
ordered_labels, metric)
else:
aggfunc = _AggFunc(metric_to_func[metric])
matrix_total = pd.crosstab(tabdf1,
tabdf2,
rownames=[feat1],
colnames=[feat2])
matrix_error = pd.crosstab(tabdf1_err,
tabdf2_err,
rownames=[feat1],
colnames=[feat2],
values=list(
zip(df_err[TRUE_Y],
df_err[PRED_Y])),
aggfunc=aggfunc._agg_func_pair)
fill_matrix_nulls(matrix_total, aggfunc._fill_na_value())
fill_matrix_nulls(matrix_error, aggfunc._fill_na_value())
matrix = matrix_2d(categories1, categories2, matrix_total,
matrix_error, metric)
else:
feat1 = dataset_sub_names[0]
unique_count1 = len(df[feat1].unique())
f1_is_cat = False
if analyzer.categorical_features is not None:
f1_is_cat = feat1 in analyzer.categorical_features
if unique_count1 > num_bins and not f1_is_cat:
cutdf = bin_data(df,
feat1,
num_bins,
quantile_binning=quantile_binning)
num_categories = len(cutdf.cat.categories)
bin_range = range(num_categories)
if len(cutdf.cat.categories) < num_bins:
warn_duplicate_edges(feat1)
catr = cutdf.cat.rename_categories(bin_range)
catn, counts = np.unique(catr.to_numpy(), return_counts=True)
# fix counts to include skipped categories
fix_counts = []
counts_idx = 0
for idx, catdf in enumerate(cutdf.cat.categories):
if idx not in catn:
fix_counts.append(0)
else:
fix_counts.append(counts[counts_idx])
counts_idx += 1
counts = fix_counts
cut_err = bin_data(df_err,
feat1,
cutdf.cat.categories,
quantile_binning=quantile_binning)
catr_err = cut_err.cat.rename_categories(bin_range)
val_err, counts_err = np.unique(catr_err.to_numpy(),
return_counts=True)
val_err = cut_err.cat.categories[val_err]
categories = cutdf.cat.categories
else:
categories, counts = np.unique(df[feat1].to_numpy(),
return_counts=True)
val_err, counts_err = np.unique(df_err[feat1].to_numpy(),
return_counts=True)
cut_err = df_err
# Compute the given metric for each group, if not using error rate
if metric != Metrics.ERROR_RATE:
if is_multi_agg_metric(metric):
ordered_labels = get_ordered_classes(analyzer.classes, true_y,
pred_y)
aggfunc = _MultiMetricAggFunc(metric_to_func[metric],
ordered_labels, metric)
else:
aggfunc = _AggFunc(metric_to_func[metric])
cutdf_err = pd.DataFrame(cut_err)
cutdf_err['metric_values'] = list(
zip(df_err[TRUE_Y], df_err[PRED_Y]))
grouped = cutdf_err.groupby([feat1])
agg_func = {'metric_values': aggfunc._agg_func_grouped}
counts_err = grouped.agg(agg_func)
counts_err = counts_err.values.ravel()
matrix = matrix_1d(categories, val_err, counts, counts_err, metric)
return matrix
def compute_json_error_tree(analyzer,
features,
filters,
composite_filters,
max_depth=DEFAULT_MAX_DEPTH,
num_leaves=DEFAULT_NUM_LEAVES):
# Fit a surrogate model on errors
if max_depth is None:
max_depth = DEFAULT_MAX_DEPTH
if num_leaves is None:
num_leaves = DEFAULT_NUM_LEAVES
is_model_analyzer = hasattr(analyzer, MODEL)
if is_model_analyzer:
filtered_df = filter_from_cohort(analyzer.dataset,
filters,
composite_filters,
analyzer.feature_names,
analyzer.true_y,
analyzer.categorical_features,
analyzer.categories)
else:
filtered_df = filter_from_cohort(analyzer.dataset,
filters,
composite_filters,
analyzer.feature_names,
analyzer.true_y,
analyzer.categorical_features,
analyzer.categories,
analyzer.pred_y)
row_index = filtered_df[ROW_INDEX]
true_y = filtered_df[TRUE_Y]
dropped_cols = [TRUE_Y, ROW_INDEX]
if not is_model_analyzer:
pred_y = filtered_df[PRED_Y]
dropped_cols.append(PRED_Y)
input_data = filtered_df.drop(columns=dropped_cols)
is_pandas = isinstance(analyzer.dataset, pd.DataFrame)
if is_pandas:
true_y = true_y.to_numpy()
else:
input_data = input_data.to_numpy()
if is_model_analyzer:
pred_y = analyzer.model.predict(input_data)
if analyzer.model_task == ModelTask.CLASSIFICATION:
diff = pred_y != true_y
else:
diff = pred_y - true_y
if not isinstance(diff, np.ndarray):
diff = np.array(diff)
if not isinstance(pred_y, np.ndarray):
pred_y = np.array(pred_y)
if not isinstance(true_y, np.ndarray):
true_y = np.array(true_y)
indexes = []
for feature in features:
indexes.append(analyzer.feature_names.index(feature))
if is_pandas:
input_data = input_data.to_numpy()
if analyzer.categorical_features:
# Inplace replacement of columns
for idx, c_i in enumerate(analyzer.categorical_indexes):
input_data[:, c_i] = analyzer.string_indexed_data[row_index, idx]
dataset_sub_features = input_data[:, indexes]
dataset_sub_names = np.array(analyzer.feature_names)[np.array(indexes)]
dataset_sub_names = list(dataset_sub_names)
categorical_info = get_categorical_info(analyzer,
dataset_sub_names)
cat_ind_reindexed, categories_reindexed = categorical_info
surrogate = create_surrogate_model(analyzer,
dataset_sub_features,
diff,
max_depth,
num_leaves,
cat_ind_reindexed)
filtered_indexed_df = pd.DataFrame(dataset_sub_features,
columns=dataset_sub_names)
filtered_indexed_df[DIFF] = diff
filtered_indexed_df[TRUE_Y] = true_y
filtered_indexed_df[PRED_Y] = pred_y
model_json = surrogate._Booster.dump_model()
tree_structure = model_json["tree_info"][0]['tree_structure']
max_split_index = get_max_split_index(tree_structure) + 1
json_tree = traverse(filtered_indexed_df,
tree_structure,
max_split_index,
(categories_reindexed,
cat_ind_reindexed),
[],
dataset_sub_names,
metric=analyzer.metric)
return json_tree
def compute_error_tree(analyzer,
features,
filters,
composite_filters,
max_depth=DEFAULT_MAX_DEPTH,
num_leaves=DEFAULT_NUM_LEAVES,
min_child_samples=DEFAULT_MIN_CHILD_SAMPLES):
"""Computes the error tree for the given dataset.
:param analyzer: The error analyzer containing the categorical
features and categories for the full dataset.
:type analyzer: BaseAnalyzer
:param features: The features to train the surrogate model on.
:type features: numpy.ndarray or pandas.DataFrame
:param filters: The filters to apply to the dataset.
:type filters: numpy.ndarray or pandas.DataFrame
:param composite_filters: The composite filters to apply to the dataset.
:type composite_filters: numpy.ndarray or pandas.DataFrame
:param max_depth: The maximum depth of the surrogate tree trained
on errors.
:type max_depth: int
:param num_leaves: The number of leaves of the surrogate tree
trained on errors.
:type num_leaves: int
:param min_child_samples: The minimal number of data required to
create one leaf.
:return: The tree representation as a list of nodes.
:rtype: list[dict[str, str]]
"""
# Fit a surrogate model on errors
if max_depth is None:
max_depth = DEFAULT_MAX_DEPTH
if num_leaves is None:
num_leaves = DEFAULT_NUM_LEAVES
if min_child_samples is None:
min_child_samples = DEFAULT_MIN_CHILD_SAMPLES
filtered_df = filter_from_cohort(analyzer, filters, composite_filters)
if filtered_df.shape[0] == 0:
return create_empty_node(analyzer.metric)
is_model_analyzer = hasattr(analyzer, MODEL)
indexes = []
for feature in features:
indexes.append(analyzer.feature_names.index(feature))
dataset_sub_names = np.array(analyzer.feature_names)[np.array(indexes)]
dataset_sub_names = list(dataset_sub_names)
if not is_spark(filtered_df):
booster, filtered_indexed_df, cat_info = get_surrogate_booster_local(
filtered_df, analyzer, is_model_analyzer, indexes,
dataset_sub_names, max_depth, num_leaves, min_child_samples)
cat_ind_reindexed, categories_reindexed = cat_info
else:
booster, filtered_indexed_df = get_surrogate_booster_pyspark(
filtered_df, analyzer, max_depth, num_leaves, min_child_samples)
cat_ind_reindexed = []
categories_reindexed = []
dumped_model = booster.dump_model()
tree_structure = dumped_model["tree_info"][0]['tree_structure']
max_split_index = get_max_split_index(tree_structure) + 1
cache_subtree_features(tree_structure, dataset_sub_names)
tree = traverse(filtered_indexed_df,
tree_structure,
max_split_index, (categories_reindexed, cat_ind_reindexed),
[],
dataset_sub_names,
metric=analyzer.metric,
classes=analyzer.classes)
return tree