def command(job_id, split_desc, core_params, preprocessing_folder, model_folder, computation_parameters): # LOADING INFO # model_handler = PredictionModelInformationHandler(split_desc, core_params, preprocessing_folder, model_folder) test_df = model_handler.get_test_df() # COMPUTING SUBPOPULATION # col_name = get_computation_parameter("column", computation_parameters) col_type = get_type_of_column(col_name, model_handler) if col_type == "CATEGORY": value = get_computation_parameter("value", computation_parameters) subpop_df = test_df[test_df[col_name] == value] else: raise NotImplementedError("Not implemented yet :-(") # COMPUTING NEW METRICS ON SUBPOP # prediction_type = model_handler.get_prediction_type() if prediction_type == constants.BINARY_CLASSIFICATION: results = compute_binary_subpopulation_metrics(subpop_df, model_handler) else: raise NotImplementedError("Not implemented yet :-(") dkujson.dump_to_filepath(osp.join(model_folder, "subpop.json"), results) return "ok"
def get_drift_metrics(): try: model_id = request.args.get('model_id') version_id = request.args.get('version_id') test_set = request.args.get('test_set') new_test_df = dataiku.Dataset(test_set).get_dataframe( bool_as_str=True, limit=ModelDriftConstants.MAX_NUM_ROW) fmi = get_webapp_config().get("trainedModelFullModelId") if fmi is None: model = dataiku.Model(model_id) model_handler = get_model_handler(model, version_id=version_id) model_accessor = ModelAccessor(model_handler) else: original_model_handler = PredictionModelInformationHandler.from_full_model_id( fmi) model_accessor = ModelAccessor(original_model_handler) drifter = DriftAnalyzer() drifter.fit(new_test_df, model_accessor=model_accessor) return json.dumps(drifter.get_drift_metrics_for_webapp(), allow_nan=False, default=convert_numpy_int64_to_int) except: logger.error(traceback.format_exc()) return traceback.format_exc(), 500
def get_histograms(model_id, version_id, advantageous_outcome, sensitive_column): fmi = get_webapp_config().get("trainedModelFullModelId") if fmi is None: model = dataiku.Model(model_id) model_handler = get_model_handler(model, version_id=version_id) model_accessor = ModelAccessor(model_handler) else: original_model_handler = PredictionModelInformationHandler.from_full_model_id( fmi) model_accessor = ModelAccessor(original_model_handler) raw_test_df = model_accessor.get_original_test_df() test_df = raw_test_df.dropna(subset=[sensitive_column]) target_variable = model_accessor.get_target_variable() y_true = test_df.loc[:, target_variable] pred_df = model_accessor.predict(test_df) y_pred = pred_df.loc[:, DkuWebappConstants.PREDICTION] advantageous_outcome_proba_col = 'proba_{}'.format(advantageous_outcome) y_pred_proba = pred_df.loc[:, advantageous_outcome_proba_col] sensitive_feature_values = test_df[sensitive_column] return get_histogram_data(y_true, y_pred, y_pred_proba, advantageous_outcome, sensitive_feature_values)
def get_value_list(model_id, version_id, column): try: if column == 'undefined' or column == 'null': raise ValueError('Please choose a column.') fmi = get_webapp_config().get("trainedModelFullModelId") if fmi is None: model = dataiku.Model(model_id) model_handler = get_model_handler(model, version_id=version_id) model_accessor = ModelAccessor(model_handler) else: original_model_handler = PredictionModelInformationHandler.from_full_model_id( fmi) model_accessor = ModelAccessor(original_model_handler) test_df = model_accessor.get_original_test_df() value_list = test_df[column].unique().tolist( ) # should check for categorical variables ? filtered_value_list = remove_nan_from_list(value_list) if len(filtered_value_list) > DkuWebappConstants.MAX_NUM_CATEGORIES: raise ValueError( 'Column "{2}" is either of numerical type or has too many categories ({0}). Max {1} are allowed.' .format(len(filtered_value_list), DkuWebappConstants.MAX_NUM_CATEGORIES, column)) return simplejson.dumps(filtered_value_list, ignore_nan=True, default=convert_numpy_int64_to_int) except: logger.error("When trying to call get-value-list endpoint: {}.".format( traceback.format_exc())) return "{}Check backend log for more details.".format( traceback.format_exc()), 500
def check_model_type(model_id, version_id): try: fmi = get_webapp_config().get("trainedModelFullModelId") if fmi is None: model = dataiku.Model(model_id) model_handler = get_model_handler(model, version_id=version_id) model_accessor = ModelAccessor(model_handler) else: original_model_handler = PredictionModelInformationHandler.from_full_model_id( fmi) model_accessor = ModelAccessor(original_model_handler) if model_accessor.get_prediction_type() in [ DkuModelAccessorConstants.REGRRSSION_TYPE, DkuModelAccessorConstants.CLUSTERING_TYPE ]: raise ValueError( 'Model Fairness Report only supports binary classification model.' ) return 'ok' except: logger.error( "When trying to call check-model-type endpoint: {}.".format( traceback.format_exc())) return "{}Check backend log for more details.".format( traceback.format_exc()), 500
def get_outcome_list(model_id, version_id): try: fmi = get_webapp_config().get("trainedModelFullModelId") if fmi is None: model = dataiku.Model(model_id) model_handler = get_model_handler(model, version_id=version_id) model_accessor = ModelAccessor(model_handler) else: original_model_handler = PredictionModelInformationHandler.from_full_model_id( fmi) model_accessor = ModelAccessor(original_model_handler) # note: sometimes when the dataset is very unbalanced, the original_test_df does not have all the target values test_df = model_accessor.get_original_test_df() target = model_accessor.get_target_variable() outcome_list = test_df[target].unique().tolist() filtered_outcome_list = remove_nan_from_list(outcome_list) return simplejson.dumps(filtered_outcome_list, ignore_nan=True, default=convert_numpy_int64_to_int) except: logger.error( "When trying to call get-outcome-list endpoint: {}.".format( traceback.format_exc())) return "{}Check backend log for more details.".format( traceback.format_exc()), 500
def get_model_info(): try: logger.info('Retrieving model data...') fmi = get_webapp_config().get("trainedModelFullModelId") if fmi is None: model = Model(get_webapp_config()["modelId"]) version_id = get_webapp_config().get("versionId") original_model_handler = get_model_handler(model, version_id) else: original_model_handler = PredictionModelInformationHandler.from_full_model_id( fmi) stressor.model_accessor = ModelAccessor(original_model_handler) return jsonify( target_classes=stressor.model_accessor.get_target_classes(), pred_type=stressor.model_accessor.get_prediction_type(), features={ feature: preprocessing["type"] for (feature, preprocessing ) in stressor.model_accessor.get_per_feature().items() if preprocessing["role"] == "INPUT" }, metric=stressor.model_accessor.get_evaluation_metric()) except: logger.error(traceback.format_exc()) return traceback.format_exc(), 500
def get_model_handler(model, version_id=None): try: params = model.get_predictor(version_id).params return PredictionModelInformationHandler( params.split_desc, params.core_params, params.model_folder, params.model_folder ) except Exception as e: if "ordinal not in range(128)" in safe_str(e): raise Exception("Model stress test only supports models built with Python 3. This one was built with Python 2.") from None else: raise e
def _get_model_info_handler(saved_model_version_id): infos = saved_model_version_id.split("-") if len(infos) != 4 or infos[0] != "S": raise ValueError("Invalid saved model id") pkey = infos[1] model_id = infos[2] version_id = infos[3] datadir_path = os.environ['DIP_HOME'] version_folder = os.path.join(datadir_path, "saved_models", pkey, model_id, "versions", version_id) # Loading and resolving paths in split_desc split_folder = os.path.join(version_folder, "split") with open(os.path.join(split_folder, "split.json")) as split_file: split_desc = json.load(split_file) path_field_names = ["trainPath", "testPath", "fullPath"] for field_name in path_field_names: if split_desc.get(field_name, None) is not None: split_desc[field_name] = os.path.join(split_folder, split_desc[field_name]) with open(os.path.join(version_folder, "core_params.json")) as core_params_file: core_params = json.load(core_params_file) try: return PredictionModelInformationHandler(split_desc, core_params, version_folder, version_folder) except Exception as e: from future.utils import raise_ if "ordinal not in range(128)" in str(e): raise_( Exception, "The plugin is using a python3 code-env, cannot load a python2 model.", sys.exc_info()[2]) elif str(e) == "non-string names in Numpy dtype unpickling": raise_( Exception, "The plugin is using a python2 code-env, cannot load a python3 model.", sys.exc_info()[2]) else: raise_(Exception, "Fail to load saved model: {}".format(e), sys.exc_info()[2])
def get_feature_list(model_id, version_id): try: fmi = get_webapp_config().get("trainedModelFullModelId") if fmi is None: model = dataiku.Model(model_id) model_handler = get_model_handler(model, version_id=version_id) model_accessor = ModelAccessor(model_handler) else: original_model_handler = PredictionModelInformationHandler.from_full_model_id( fmi) model_accessor = ModelAccessor(original_model_handler) column_list = model_accessor.get_selected_and_rejected_features() return simplejson.dumps(column_list, ignore_nan=True, default=convert_numpy_int64_to_int) except: logger.error( "When trying to call get-feature-list endpoint: {}.".format( traceback.format_exc())) return "{}Check backend log for more details.".format( traceback.format_exc()), 500
def get_original_model_info(): try: fmi = get_webapp_config().get("trainedModelFullModelId") if fmi is None: model = Model(get_webapp_config()["modelId"]) version_id = get_webapp_config().get("versionId") original_model_handler = get_model_handler(model, version_id) name = model.get_name() else: original_model_handler = PredictionModelInformationHandler.from_full_model_id( fmi) name = DSSMLTask.from_full_model_id( api_client(), fmi).get_trained_model_snippet(fmi).get("userMeta", {}).get("name", fmi) handler.set_error_analyzer(original_model_handler) return jsonify(modelName=name, isRegression='REGRESSION' in original_model_handler.get_prediction_type()) except: LOGGER.error(traceback.format_exc()) return traceback.format_exc(), 500
def get_metrics(model_id, version_id, advantageous_outcome, sensitive_column, reference_group): fmi = get_webapp_config().get("trainedModelFullModelId") if fmi is None: model = dataiku.Model(model_id) model_handler = get_model_handler(model, version_id=version_id) model_accessor = ModelAccessor(model_handler) else: original_model_handler = PredictionModelInformationHandler.from_full_model_id( fmi) model_accessor = ModelAccessor(original_model_handler) test_df = model_accessor.get_original_test_df() target_variable = model_accessor.get_target_variable() test_df.dropna(subset=[sensitive_column, target_variable], how='any', inplace=True) y_true = test_df.loc[:, target_variable] pred_df = model_accessor.predict(test_df) y_pred = pred_df.loc[:, DkuWebappConstants.PREDICTION] try: # check whether or not the column can be casted to int if np.array_equal(test_df[sensitive_column], test_df[sensitive_column].astype(int)): test_df[sensitive_column] = test_df[sensitive_column].astype(int) if test_df[sensitive_column].dtypes == int: reference_group = int(reference_group) if test_df[sensitive_column].dtypes == float: reference_group = float(reference_group) except Exception as e: logger.info('Sensitive column can not be casted to int. ', e) pass sensitive_feature_values = test_df[sensitive_column] model_report = ModelFairnessMetricReport(y_true, y_pred, sensitive_feature_values, advantageous_outcome) population_names = sensitive_feature_values.unique() metric_dct = {} disparity_dct = {} for metric_func in ModelFairnessMetric.get_available_metric_functions(): metric_summary = model_report.compute_metric_per_group( metric_function=metric_func) metric_dct[metric_func.__name__] = metric_summary.get( DkuFairnessConstants.BY_GROUP) metric_diff = model_report.compute_group_difference_from_summary( metric_summary, reference_group=reference_group) v = np.array( list(metric_diff.get( DkuFairnessConstants.BY_GROUP).values())).reshape( 1, -1).squeeze() v_without_nan = [x for x in v if not np.isnan(x)] if len(v_without_nan) > 0: max_disparity = max(v_without_nan, key=abs) disparity_dct[metric_func.__name__] = max_disparity else: disparity_dct[metric_func.__name__] = 'N/A' # for display purpose populations = [] for name in population_names: dct = { DkuWebappConstants.NAME: name, DkuWebappConstants.SIZE: len(test_df[test_df[sensitive_column] == name]) } for m, v in metric_dct.items(): # the following strings are used only here, too lazy to turn them into constant variables if m == 'demographic_parity': dct['positive_rate'] = v[name] if m == 'equalized_odds': dct['true_positive_rate'], dct['false_positive_rate'] = v[name] if m == 'predictive_rate_parity': dct['positive_predictive_value'] = v[name] # make sure that NaN is replaced by a string (a dot here), for display purpose for k, v in dct.items(): if not isinstance(v, str) and np.isnan(v): dct[k] = '.' populations.append(dct) label_list = model_report.get_label_list() sorted_populations = sorted( populations, key=lambda population: population[DkuWebappConstants.SIZE], reverse=True) return sorted_populations, disparity_dct, label_list
def compute(job_id, split_desc, core_params, preprocessing_folder, model_folder, computation_params): if computation_params is None or "features_to_compute" not in computation_params: raise Exception( "'computation_params' should contains a key 'features_to_compute'") model_handler = PredictionModelInformationHandler(split_desc, core_params, preprocessing_folder, model_folder) features_to_compute = computation_params["features_to_compute"] if model_handler.is_kfolding(): df = model_handler.get_full_df() else: df = model_handler.get_test_df() progress = PartialDependenciesProgress(job_id, len(features_to_compute)) saver = PartialDependenciesSaver(model_folder, split_desc["schema"]) computer = PartialDependencyComputer( df, model_handler.get_prediction_type(), model_handler.predict, progress, model_handler.get_sample_weight_variable()) for index, feature_name in enumerate(features_to_compute): drop_missing = model_handler.get_per_feature_col(feature_name).get( "missing_handling") == "DROP_ROW" feature_type = model_handler.get_type_of_column(feature_name) is_dummified = False category_possible_value = None if feature_type == 'CATEGORY': # nan values are replaced by a fake one because neither a scale nor a distribution can be computed with nan feature_values = df[feature_name].fillna( constants.FILL_NA_VALUE).values is_dummified = model_handler.is_column_dummified(feature_name) category_possible_value = model_handler.category_possible_values( feature_name) else: feature_values = df[feature_name].values pd_feature = PartialDependencyFeature(feature_type, feature_values, feature_name, is_dummified, category_possible_value, drop_missing) result = computer.compute(pd_feature) saver.save(result) progress.set_percentage((index + 1) * 100 / len(features_to_compute), no_fail=False)