def get_model_info():
try:
logger.info('Retrieving model data...')
fmi = get_webapp_config().get("trainedModelFullModelId")
if fmi is None:
model = Model(get_webapp_config()["modelId"])
version_id = get_webapp_config().get("versionId")
original_model_handler = get_model_handler(model, version_id)
else:
original_model_handler = PredictionModelInformationHandler.from_full_model_id(
fmi)
stressor.model_accessor = ModelAccessor(original_model_handler)
return jsonify(
target_classes=stressor.model_accessor.get_target_classes(),
pred_type=stressor.model_accessor.get_prediction_type(),
features={
feature: preprocessing["type"]
for (feature, preprocessing
) in stressor.model_accessor.get_per_feature().items()
if preprocessing["role"] == "INPUT"
},
metric=stressor.model_accessor.get_evaluation_metric())
except:
logger.error(traceback.format_exc())
return traceback.format_exc(), 500
def get_drift_metrics():
try:
model_id = request.args.get('model_id')
version_id = request.args.get('version_id')
test_set = request.args.get('test_set')
new_test_df = dataiku.Dataset(test_set).get_dataframe(
bool_as_str=True, limit=ModelDriftConstants.MAX_NUM_ROW)
fmi = get_webapp_config().get("trainedModelFullModelId")
if fmi is None:
model = dataiku.Model(model_id)
model_handler = get_model_handler(model, version_id=version_id)
model_accessor = ModelAccessor(model_handler)
else:
original_model_handler = PredictionModelInformationHandler.from_full_model_id(
fmi)
model_accessor = ModelAccessor(original_model_handler)
drifter = DriftAnalyzer()
drifter.fit(new_test_df, model_accessor=model_accessor)
return json.dumps(drifter.get_drift_metrics_for_webapp(),
allow_nan=False,
default=convert_numpy_int64_to_int)
except:
logger.error(traceback.format_exc())
return traceback.format_exc(), 500
def get_histograms(model_id, version_id, advantageous_outcome,
sensitive_column):
fmi = get_webapp_config().get("trainedModelFullModelId")
if fmi is None:
model = dataiku.Model(model_id)
model_handler = get_model_handler(model, version_id=version_id)
model_accessor = ModelAccessor(model_handler)
else:
original_model_handler = PredictionModelInformationHandler.from_full_model_id(
fmi)
model_accessor = ModelAccessor(original_model_handler)
raw_test_df = model_accessor.get_original_test_df()
test_df = raw_test_df.dropna(subset=[sensitive_column])
target_variable = model_accessor.get_target_variable()
y_true = test_df.loc[:, target_variable]
pred_df = model_accessor.predict(test_df)
y_pred = pred_df.loc[:, DkuWebappConstants.PREDICTION]
advantageous_outcome_proba_col = 'proba_{}'.format(advantageous_outcome)
y_pred_proba = pred_df.loc[:, advantageous_outcome_proba_col]
sensitive_feature_values = test_df[sensitive_column]
return get_histogram_data(y_true, y_pred, y_pred_proba,
advantageous_outcome, sensitive_feature_values)
def get_value_list(model_id, version_id, column):
try:
if column == 'undefined' or column == 'null':
raise ValueError('Please choose a column.')
fmi = get_webapp_config().get("trainedModelFullModelId")
if fmi is None:
model = dataiku.Model(model_id)
model_handler = get_model_handler(model, version_id=version_id)
model_accessor = ModelAccessor(model_handler)
else:
original_model_handler = PredictionModelInformationHandler.from_full_model_id(
fmi)
model_accessor = ModelAccessor(original_model_handler)
test_df = model_accessor.get_original_test_df()
value_list = test_df[column].unique().tolist(
) # should check for categorical variables ?
filtered_value_list = remove_nan_from_list(value_list)
if len(filtered_value_list) > DkuWebappConstants.MAX_NUM_CATEGORIES:
raise ValueError(
'Column "{2}" is either of numerical type or has too many categories ({0}). Max {1} are allowed.'
.format(len(filtered_value_list),
DkuWebappConstants.MAX_NUM_CATEGORIES, column))
return simplejson.dumps(filtered_value_list,
ignore_nan=True,
default=convert_numpy_int64_to_int)
except:
logger.error("When trying to call get-value-list endpoint: {}.".format(
traceback.format_exc()))
return "{}Check backend log for more details.".format(
traceback.format_exc()), 500
def check_model_type(model_id, version_id):
try:
fmi = get_webapp_config().get("trainedModelFullModelId")
if fmi is None:
model = dataiku.Model(model_id)
model_handler = get_model_handler(model, version_id=version_id)
model_accessor = ModelAccessor(model_handler)
else:
original_model_handler = PredictionModelInformationHandler.from_full_model_id(
fmi)
model_accessor = ModelAccessor(original_model_handler)
if model_accessor.get_prediction_type() in [
DkuModelAccessorConstants.REGRRSSION_TYPE,
DkuModelAccessorConstants.CLUSTERING_TYPE
]:
raise ValueError(
'Model Fairness Report only supports binary classification model.'
)
return 'ok'
except:
logger.error(
"When trying to call check-model-type endpoint: {}.".format(
traceback.format_exc()))
return "{}Check backend log for more details.".format(
traceback.format_exc()), 500
def get_outcome_list(model_id, version_id):
try:
fmi = get_webapp_config().get("trainedModelFullModelId")
if fmi is None:
model = dataiku.Model(model_id)
model_handler = get_model_handler(model, version_id=version_id)
model_accessor = ModelAccessor(model_handler)
else:
original_model_handler = PredictionModelInformationHandler.from_full_model_id(
fmi)
model_accessor = ModelAccessor(original_model_handler)
# note: sometimes when the dataset is very unbalanced, the original_test_df does not have all the target values
test_df = model_accessor.get_original_test_df()
target = model_accessor.get_target_variable()
outcome_list = test_df[target].unique().tolist()
filtered_outcome_list = remove_nan_from_list(outcome_list)
return simplejson.dumps(filtered_outcome_list,
ignore_nan=True,
default=convert_numpy_int64_to_int)
except:
logger.error(
"When trying to call get-outcome-list endpoint: {}.".format(
traceback.format_exc()))
return "{}Check backend log for more details.".format(
traceback.format_exc()), 500
def get_original_model_info():
try:
fmi = get_webapp_config().get("trainedModelFullModelId")
if fmi is None:
model = Model(get_webapp_config()["modelId"])
version_id = get_webapp_config().get("versionId")
original_model_handler = get_model_handler(model, version_id)
name = model.get_name()
else:
original_model_handler = PredictionModelInformationHandler.from_full_model_id(
fmi)
name = DSSMLTask.from_full_model_id(
api_client(),
fmi).get_trained_model_snippet(fmi).get("userMeta",
{}).get("name", fmi)
handler.set_error_analyzer(original_model_handler)
return jsonify(modelName=name,
isRegression='REGRESSION'
in original_model_handler.get_prediction_type())
except:
LOGGER.error(traceback.format_exc())
return traceback.format_exc(), 500
def get_feature_list(model_id, version_id):
try:
fmi = get_webapp_config().get("trainedModelFullModelId")
if fmi is None:
model = dataiku.Model(model_id)
model_handler = get_model_handler(model, version_id=version_id)
model_accessor = ModelAccessor(model_handler)
else:
original_model_handler = PredictionModelInformationHandler.from_full_model_id(
fmi)
model_accessor = ModelAccessor(original_model_handler)
column_list = model_accessor.get_selected_and_rejected_features()
return simplejson.dumps(column_list,
ignore_nan=True,
default=convert_numpy_int64_to_int)
except:
logger.error(
"When trying to call get-feature-list endpoint: {}.".format(
traceback.format_exc()))
return "{}Check backend log for more details.".format(
traceback.format_exc()), 500
def get_metrics(model_id, version_id, advantageous_outcome, sensitive_column,
reference_group):
fmi = get_webapp_config().get("trainedModelFullModelId")
if fmi is None:
model = dataiku.Model(model_id)
model_handler = get_model_handler(model, version_id=version_id)
model_accessor = ModelAccessor(model_handler)
else:
original_model_handler = PredictionModelInformationHandler.from_full_model_id(
fmi)
model_accessor = ModelAccessor(original_model_handler)
test_df = model_accessor.get_original_test_df()
target_variable = model_accessor.get_target_variable()
test_df.dropna(subset=[sensitive_column, target_variable],
how='any',
inplace=True)
y_true = test_df.loc[:, target_variable]
pred_df = model_accessor.predict(test_df)
y_pred = pred_df.loc[:, DkuWebappConstants.PREDICTION]
try: # check whether or not the column can be casted to int
if np.array_equal(test_df[sensitive_column],
test_df[sensitive_column].astype(int)):
test_df[sensitive_column] = test_df[sensitive_column].astype(int)
if test_df[sensitive_column].dtypes == int:
reference_group = int(reference_group)
if test_df[sensitive_column].dtypes == float:
reference_group = float(reference_group)
except Exception as e:
logger.info('Sensitive column can not be casted to int. ', e)
pass
sensitive_feature_values = test_df[sensitive_column]
model_report = ModelFairnessMetricReport(y_true, y_pred,
sensitive_feature_values,
advantageous_outcome)
population_names = sensitive_feature_values.unique()
metric_dct = {}
disparity_dct = {}
for metric_func in ModelFairnessMetric.get_available_metric_functions():
metric_summary = model_report.compute_metric_per_group(
metric_function=metric_func)
metric_dct[metric_func.__name__] = metric_summary.get(
DkuFairnessConstants.BY_GROUP)
metric_diff = model_report.compute_group_difference_from_summary(
metric_summary, reference_group=reference_group)
v = np.array(
list(metric_diff.get(
DkuFairnessConstants.BY_GROUP).values())).reshape(
1, -1).squeeze()
v_without_nan = [x for x in v if not np.isnan(x)]
if len(v_without_nan) > 0:
max_disparity = max(v_without_nan, key=abs)
disparity_dct[metric_func.__name__] = max_disparity
else:
disparity_dct[metric_func.__name__] = 'N/A' # for display purpose
populations = []
for name in population_names:
dct = {
DkuWebappConstants.NAME:
name,
DkuWebappConstants.SIZE:
len(test_df[test_df[sensitive_column] == name])
}
for m, v in metric_dct.items():
# the following strings are used only here, too lazy to turn them into constant variables
if m == 'demographic_parity':
dct['positive_rate'] = v[name]
if m == 'equalized_odds':
dct['true_positive_rate'], dct['false_positive_rate'] = v[name]
if m == 'predictive_rate_parity':
dct['positive_predictive_value'] = v[name]
# make sure that NaN is replaced by a string (a dot here), for display purpose
for k, v in dct.items():
if not isinstance(v, str) and np.isnan(v):
dct[k] = '.'
populations.append(dct)
label_list = model_report.get_label_list()
sorted_populations = sorted(
populations,
key=lambda population: population[DkuWebappConstants.SIZE],
reverse=True)
return sorted_populations, disparity_dct, label_list
from flask import request
from distutils.util import strtobool
import json
import traceback
import dataiku
from dataiku.customwebapp import get_webapp_config
from design_experiment.sample_size import min_sample_size, z_value
from helpers import save_parameters
from constants import Parameters
from dku_tools import get_output_folder
config_settings = get_webapp_config()
project_key = dataiku.default_project_key()
client = dataiku.api_client()
@app.route('/sample_size', methods=['POST'])
def get_sample_size():
try:
config = json.loads(request.data)
baseline_conversion_rate = float(config.get(Parameters.BCR.value))/100
minimum_detectable_effect = float(config.get(Parameters.MDE.value))/100
alpha = 1-float(config.get(Parameters.SIG_LEVEL.value))/100
power = float(config.get(Parameters.POWER.value))/100
ratio = float(config.get(Parameters.RATIO.value))/100
reach = float(config.get(Parameters.REACH.value))/100
two_tailed = strtobool(config.get(Parameters.TAIL.value))
sample_size_A, sample_size_B = min_sample_size(baseline_conversion_rate, minimum_detectable_effect, alpha, power, ratio, two_tailed)
sample_size_A = round(sample_size_A / reach)
sample_size_B = round(sample_size_B / reach)
import dataiku
from dataiku.customwebapp import get_webapp_config
from lal.api import define_endpoints
from lal.app_configuration import prepare_datasets
from lal.classifiers.image_object_classifier import ImageObjectClassifier
from lal.handlers.dataiku_lal_handler import DataikuLALHandler
config = get_webapp_config()
labels_schema = [{"name": "path", "type": "string"}]
prepare_datasets(config, labels_schema)
unlabeled_mf = dataiku.Folder(config["unlabeled"])
queries_df = dataiku.Dataset(
config["queries_ds"]).get_dataframe() if "queries_ds" in config else None
define_endpoints(
app,
DataikuLALHandler(ImageObjectClassifier(unlabeled_mf, queries_df, config),
config))
from dku_idtb_decision_tree.tree import Tree
from dku_idtb_decision_tree.tree_factory import TreeFactory
from dku_idtb_decision_tree.node import Node
from dku_idtb_decision_tree.autosplit import autosplit
from dku_idtb_compatibility.utils import safe_str, safe_write_json
from dataiku.core.dkujson import DKUJSONEncoder
app.json_encoder = DKUJSONEncoder
logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO,
format="IDTB %(levelname)s - %(message)s")
# initialization of the backend
try:
folder_name = get_webapp_config()["input_folder"]
except KeyError:
raise SystemError(
"No folder has been chosen in the settings of the webapp")
folder = dataiku.Folder(folder_name)
factory = TreeFactory()
#cache = {}
@app.route("/get-datasets")
def get_datasets():
try:
return jsonify(datasets=dataiku.Dataset.list())
except:
