def get_value_list(model_id, version_id, column):
try:
if column == 'undefined' or column == 'null':
raise ValueError('Please choose a column.')
fmi = get_webapp_config().get("trainedModelFullModelId")
if fmi is None:
model = dataiku.Model(model_id)
model_handler = get_model_handler(model, version_id=version_id)
model_accessor = ModelAccessor(model_handler)
else:
original_model_handler = PredictionModelInformationHandler.from_full_model_id(
fmi)
model_accessor = ModelAccessor(original_model_handler)
test_df = model_accessor.get_original_test_df()
value_list = test_df[column].unique().tolist(
) # should check for categorical variables ?
filtered_value_list = remove_nan_from_list(value_list)
if len(filtered_value_list) > DkuWebappConstants.MAX_NUM_CATEGORIES:
raise ValueError(
'Column "{2}" is either of numerical type or has too many categories ({0}). Max {1} are allowed.'
.format(len(filtered_value_list),
DkuWebappConstants.MAX_NUM_CATEGORIES, column))
return simplejson.dumps(filtered_value_list,
ignore_nan=True,
default=convert_numpy_int64_to_int)
except:
logger.error("When trying to call get-value-list endpoint: {}.".format(
traceback.format_exc()))
return "{}Check backend log for more details.".format(
traceback.format_exc()), 500
def check_model_type(model_id, version_id):
try:
fmi = get_webapp_config().get("trainedModelFullModelId")
if fmi is None:
model = dataiku.Model(model_id)
model_handler = get_model_handler(model, version_id=version_id)
model_accessor = ModelAccessor(model_handler)
else:
original_model_handler = PredictionModelInformationHandler.from_full_model_id(
fmi)
model_accessor = ModelAccessor(original_model_handler)
if model_accessor.get_prediction_type() in [
DkuModelAccessorConstants.REGRRSSION_TYPE,
DkuModelAccessorConstants.CLUSTERING_TYPE
]:
raise ValueError(
'Model Fairness Report only supports binary classification model.'
)
return 'ok'
except:
logger.error(
"When trying to call check-model-type endpoint: {}.".format(
traceback.format_exc()))
return "{}Check backend log for more details.".format(
traceback.format_exc()), 500
def get_params(config):
range_mode = config.get('range_mode')
if config.get('input_mode') == 'dataset':
df_ref = dataiku.Dataset(
config.get("ds_ref")).get_dataframe(bool_as_str=True)
columns = [col for col in config.get("columns_dataset") if col != '']
columns_not_in_df_ref = set(columns) - set(df_ref.columns)
if len(columns_not_in_df_ref) > 0:
raise ValueError(
'The following chosen columns are not in the reference dataset: {}. Please remove them from the list of columns to check.'
.format(' ,'.join(list(columns_not_in_df_ref))))
else:
model_ref = config.get('model_ref')
if model_ref is None:
raise ValueError('Please choose a reference model.')
model = dataiku.Model(model_ref)
model_handler = get_model_handler(model)
model_accessor = ModelAccessor(model_handler)
df_ref = model_accessor.get_train_df()
selected_features = model_accessor.get_selected_features()
chosen_columns = [
col for col in config.get("columns_model") if col != ''
]
if len(chosen_columns) > 0:
columns = chosen_columns
features_not_in_model = list(set(columns) - set(selected_features))
if len(features_not_in_model) > 0:
raise ValueError(
'The following chosen columns are not used in the model: {}. Please remove them from the list of columns to check.'
.format(' ,'.join(features_not_in_model)))
else:
columns = selected_features
return df_ref, columns, range_mode
def get_histograms(model_id, version_id, advantageous_outcome,
sensitive_column):
fmi = get_webapp_config().get("trainedModelFullModelId")
if fmi is None:
model = dataiku.Model(model_id)
model_handler = get_model_handler(model, version_id=version_id)
model_accessor = ModelAccessor(model_handler)
else:
original_model_handler = PredictionModelInformationHandler.from_full_model_id(
fmi)
model_accessor = ModelAccessor(original_model_handler)
raw_test_df = model_accessor.get_original_test_df()
test_df = raw_test_df.dropna(subset=[sensitive_column])
target_variable = model_accessor.get_target_variable()
y_true = test_df.loc[:, target_variable]
pred_df = model_accessor.predict(test_df)
y_pred = pred_df.loc[:, DkuWebappConstants.PREDICTION]
advantageous_outcome_proba_col = 'proba_{}'.format(advantageous_outcome)
y_pred_proba = pred_df.loc[:, advantageous_outcome_proba_col]
sensitive_feature_values = test_df[sensitive_column]
return get_histogram_data(y_true, y_pred, y_pred_proba,
advantageous_outcome, sensitive_feature_values)
def get_outcome_list(model_id, version_id):
try:
fmi = get_webapp_config().get("trainedModelFullModelId")
if fmi is None:
model = dataiku.Model(model_id)
model_handler = get_model_handler(model, version_id=version_id)
model_accessor = ModelAccessor(model_handler)
else:
original_model_handler = PredictionModelInformationHandler.from_full_model_id(
fmi)
model_accessor = ModelAccessor(original_model_handler)
# note: sometimes when the dataset is very unbalanced, the original_test_df does not have all the target values
test_df = model_accessor.get_original_test_df()
target = model_accessor.get_target_variable()
outcome_list = test_df[target].unique().tolist()
filtered_outcome_list = remove_nan_from_list(outcome_list)
return simplejson.dumps(filtered_outcome_list,
ignore_nan=True,
default=convert_numpy_int64_to_int)
except:
logger.error(
"When trying to call get-outcome-list endpoint: {}.".format(
traceback.format_exc()))
return "{}Check backend log for more details.".format(
traceback.format_exc()), 500
def get_feature_list(model_id, version_id):
try:
fmi = get_webapp_config().get("trainedModelFullModelId")
if fmi is None:
model = dataiku.Model(model_id)
model_handler = get_model_handler(model, version_id=version_id)
model_accessor = ModelAccessor(model_handler)
else:
original_model_handler = PredictionModelInformationHandler.from_full_model_id(
fmi)
model_accessor = ModelAccessor(original_model_handler)
column_list = model_accessor.get_selected_and_rejected_features()
return simplejson.dumps(column_list,
ignore_nan=True,
default=convert_numpy_int64_to_int)
except:
logger.error(
"When trying to call get-feature-list endpoint: {}.".format(
traceback.format_exc()))
return "{}Check backend log for more details.".format(
traceback.format_exc()), 500
def get_metrics(model_id, version_id, advantageous_outcome, sensitive_column,
reference_group):
fmi = get_webapp_config().get("trainedModelFullModelId")
if fmi is None:
model = dataiku.Model(model_id)
model_handler = get_model_handler(model, version_id=version_id)
model_accessor = ModelAccessor(model_handler)
else:
original_model_handler = PredictionModelInformationHandler.from_full_model_id(
fmi)
model_accessor = ModelAccessor(original_model_handler)
test_df = model_accessor.get_original_test_df()
target_variable = model_accessor.get_target_variable()
test_df.dropna(subset=[sensitive_column, target_variable],
how='any',
inplace=True)
y_true = test_df.loc[:, target_variable]
pred_df = model_accessor.predict(test_df)
y_pred = pred_df.loc[:, DkuWebappConstants.PREDICTION]
try: # check whether or not the column can be casted to int
if np.array_equal(test_df[sensitive_column],
test_df[sensitive_column].astype(int)):
test_df[sensitive_column] = test_df[sensitive_column].astype(int)
if test_df[sensitive_column].dtypes == int:
reference_group = int(reference_group)
if test_df[sensitive_column].dtypes == float:
reference_group = float(reference_group)
except Exception as e:
logger.info('Sensitive column can not be casted to int. ', e)
pass
sensitive_feature_values = test_df[sensitive_column]
model_report = ModelFairnessMetricReport(y_true, y_pred,
sensitive_feature_values,
advantageous_outcome)
population_names = sensitive_feature_values.unique()
metric_dct = {}
disparity_dct = {}
for metric_func in ModelFairnessMetric.get_available_metric_functions():
metric_summary = model_report.compute_metric_per_group(
metric_function=metric_func)
metric_dct[metric_func.__name__] = metric_summary.get(
DkuFairnessConstants.BY_GROUP)
metric_diff = model_report.compute_group_difference_from_summary(
metric_summary, reference_group=reference_group)
v = np.array(
list(metric_diff.get(
DkuFairnessConstants.BY_GROUP).values())).reshape(
1, -1).squeeze()
v_without_nan = [x for x in v if not np.isnan(x)]
if len(v_without_nan) > 0:
max_disparity = max(v_without_nan, key=abs)
disparity_dct[metric_func.__name__] = max_disparity
else:
disparity_dct[metric_func.__name__] = 'N/A' # for display purpose
populations = []
for name in population_names:
dct = {
DkuWebappConstants.NAME:
name,
DkuWebappConstants.SIZE:
len(test_df[test_df[sensitive_column] == name])
}
for m, v in metric_dct.items():
# the following strings are used only here, too lazy to turn them into constant variables
if m == 'demographic_parity':
dct['positive_rate'] = v[name]
if m == 'equalized_odds':
dct['true_positive_rate'], dct['false_positive_rate'] = v[name]
if m == 'predictive_rate_parity':
dct['positive_predictive_value'] = v[name]
# make sure that NaN is replaced by a string (a dot here), for display purpose
for k, v in dct.items():
if not isinstance(v, str) and np.isnan(v):
dct[k] = '.'
populations.append(dct)
label_list = model_report.get_label_list()
sorted_populations = sorted(
populations,
key=lambda population: population[DkuWebappConstants.SIZE],
reverse=True)
return sorted_populations, disparity_dct, label_list