def test_rai_dashboard_input_adult_importances_success( self, create_rai_insights_object_classification): ri = create_rai_insights_object_classification dashboard_input = ResponsibleAIDashboardInput(ri) flask_server_prediction_output = dashboard_input.importances() self.check_success_criteria(flask_server_prediction_output)
def test_rai_dashboard_input_housing_causal_global_policy_success( self, create_rai_insights_object_regression, filters): ri = create_rai_insights_object_regression dashboard_input = ResponsibleAIDashboardInput(ri) id = ri.causal.get()[0].id post_data = (id, filters, []) flask_server_prediction_output = \ dashboard_input.get_global_causal_policy( post_data) self.check_success_criteria(flask_server_prediction_output)
def test_rai_dashboard_input_adult_on_predict_failure( self, create_rai_insights_object_classification): ri = create_rai_insights_object_classification test_data = ri.test dashboard_input = ResponsibleAIDashboardInput(ri) test_pred_data = test_data.head(1).values flask_server_prediction_output = dashboard_input.on_predict( test_pred_data) self.check_failure_criteria( flask_server_prediction_output, "Model threw exception while predicting...")
def test_model_analysis_adult(self, create_rai_insights_object_classification): ri = create_rai_insights_object_classification knn = ri.model test_data = ri.test dashboard_input = ResponsibleAIDashboardInput(ri) with patch.object(knn, "predict_proba") as predict_mock: test_pred_data = test_data.head(1).drop("Income", axis=1).values dashboard_input.on_predict(test_pred_data) assert ( predict_mock.call_args[0][0].values == test_pred_data).all()
def test_rai_dashboard_input_housing_causal_global_effects_failure( self, create_rai_insights_object_regression): ri = create_rai_insights_object_regression dashboard_input = ResponsibleAIDashboardInput(ri) id = "some_id_that_does_not_exist" post_data = (id, [], []) flask_server_prediction_output = \ dashboard_input.get_global_causal_effects( post_data) self.check_failure_criteria( flask_server_prediction_output, "Failed to generate global causal effects for cohort,")
def test_rai_dashboard_input_iris_on_predict_success( self, create_rai_insights_object_multiclass_classification): ri = create_rai_insights_object_multiclass_classification rf = ri.model test_data = ri.test dashboard_input = ResponsibleAIDashboardInput(ri) test_pred_data = test_data.head(1).drop("target", axis=1).values flask_server_prediction_output = dashboard_input.on_predict( test_pred_data) rf_prediction = rf.predict_proba(test_pred_data) assert rf_prediction is not None assert (flask_server_prediction_output['data'] == rf_prediction).all() self.check_success_criteria(flask_server_prediction_output)
def test_rai_dashboard_input_housing_causal_whatif_success( self, create_rai_insights_object_regression): ri = create_rai_insights_object_regression id = ri.causal.get()[0].id causal_whatif_test_data = ri.test.head(1).drop( "target", axis=1).to_dict(orient='records') treatment_feature = 'AveRooms' current_treatment_value = [ causal_whatif_test_data[0][treatment_feature] ] current_outcome = [ri.test.head(1)["target"].values[0]] dashboard_input = ResponsibleAIDashboardInput(ri) post_data = (id, causal_whatif_test_data, treatment_feature, current_treatment_value, current_outcome) flask_server_prediction_output = dashboard_input.causal_whatif( post_data) self.check_success_criteria(flask_server_prediction_output)
def test_rai_dashboard_input_adult_debug_ml_success( self, create_rai_insights_object_classification): ri = create_rai_insights_object_classification features = ri.test.drop("Income", axis=1).columns.tolist() filters = [] composite_filters = [] max_depth = 3 num_leaves = 3 min_child_samples = 8 metric = "Error rate" post_data = [ features, filters, composite_filters, max_depth, num_leaves, min_child_samples, metric ] dashboard_input = ResponsibleAIDashboardInput(ri) flask_server_prediction_output = dashboard_input.debug_ml(post_data) self.check_success_criteria(flask_server_prediction_output)
def test_rai_dashboard_input_adult_matrix_failure( self, create_rai_insights_object_classification): ri = create_rai_insights_object_classification features = ['Age', 'Workclass'] filters = [] composite_filters = [] quantile_binning = False num_bins = 8 metric = "Error Rate" post_data = [ features, filters, composite_filters, quantile_binning, num_bins, metric ] dashboard_input = ResponsibleAIDashboardInput(ri) flask_server_prediction_output = dashboard_input.matrix(post_data) self.check_failure_criteria( flask_server_prediction_output, "Failed to generate json matrix representation,")
def test_rai_dashboard_input_adult_matrix_success( self, create_rai_insights_object_classification): ri = create_rai_insights_object_classification features = ['Age', 'Workclass'] filters = [] composite_filters = [] quantile_binning = False num_bins = 8 metric = "Error rate" post_data = [ features, filters, composite_filters, quantile_binning, num_bins, metric ] dashboard_input = ResponsibleAIDashboardInput(ri) flask_server_prediction_output = dashboard_input.matrix(post_data) self.check_success_criteria(flask_server_prediction_output) empty_features = [None, None] post_data = [ empty_features, filters, composite_filters, quantile_binning, num_bins, metric ] dashboard_input = ResponsibleAIDashboardInput(ri) flask_server_prediction_output = dashboard_input.matrix(post_data) assert len(flask_server_prediction_output[ WidgetRequestResponseConstants.data]) == 0 self.check_success_criteria(flask_server_prediction_output)
def __init__(self, analysis: RAIInsights, public_ip=None, port=None, locale=None, cohort_list=None, **kwargs): self.input = ResponsibleAIDashboardInput(analysis, cohort_list=cohort_list) super(ResponsibleAIDashboard, self).__init__(dashboard_type="ResponsibleAI", model_data=self.input.dashboard_input, public_ip=public_ip, port=port, locale=locale, no_inline_dashboard=True, **kwargs) def predict(): data = request.get_json(force=True) return jsonify(self.input.on_predict(data)) self.add_url_rule(predict, '/predict', methods=["POST"]) def tree(): data = request.get_json(force=True) return jsonify(self.input.debug_ml(data)) self.add_url_rule(tree, '/tree', methods=["POST"]) def matrix(): data = request.get_json(force=True) return jsonify(self.input.matrix(data)) self.add_url_rule(matrix, '/matrix', methods=["POST"]) def causal_whatif(): data = request.get_json(force=True) return jsonify(self.input.causal_whatif(data)) self.add_url_rule(causal_whatif, '/causal_whatif', methods=["POST"]) def global_causal_effects(): data = request.get_json(force=True) return jsonify(self.input.get_global_causal_effects(data)) self.add_url_rule(global_causal_effects, '/global_causal_effects', methods=["POST"]) def global_causal_policy(): data = request.get_json(force=True) return jsonify(self.input.get_global_causal_policy(data)) self.add_url_rule(global_causal_policy, '/global_causal_policy', methods=["POST"]) def importances(): return jsonify(self.input.importances()) self.add_url_rule(importances, '/importances', methods=["POST"])
class ResponsibleAIDashboard(Dashboard): """The dashboard class, wraps the dashboard component. :param analysis: An object that represents an RAIInsights. :type analysis: RAIInsights :param public_ip: Optional. If running on a remote vm, the external public ip address of the VM. :type public_ip: str :param port: The port to use on locally hosted service. :type port: int :param locale: The language in which user wants to load and access the ResponsibleAI Dashboard. The default language is english ("en"). :type locale: str :param cohort_list: List of cohorts defined by the user for the dashboard. :type cohort_list: List[Cohort] """ def __init__(self, analysis: RAIInsights, public_ip=None, port=None, locale=None, cohort_list=None, **kwargs): self.input = ResponsibleAIDashboardInput(analysis, cohort_list=cohort_list) super(ResponsibleAIDashboard, self).__init__(dashboard_type="ResponsibleAI", model_data=self.input.dashboard_input, public_ip=public_ip, port=port, locale=locale, no_inline_dashboard=True, **kwargs) def predict(): data = request.get_json(force=True) return jsonify(self.input.on_predict(data)) self.add_url_rule(predict, '/predict', methods=["POST"]) def tree(): data = request.get_json(force=True) return jsonify(self.input.debug_ml(data)) self.add_url_rule(tree, '/tree', methods=["POST"]) def matrix(): data = request.get_json(force=True) return jsonify(self.input.matrix(data)) self.add_url_rule(matrix, '/matrix', methods=["POST"]) def causal_whatif(): data = request.get_json(force=True) return jsonify(self.input.causal_whatif(data)) self.add_url_rule(causal_whatif, '/causal_whatif', methods=["POST"]) def global_causal_effects(): data = request.get_json(force=True) return jsonify(self.input.get_global_causal_effects(data)) self.add_url_rule(global_causal_effects, '/global_causal_effects', methods=["POST"]) def global_causal_policy(): data = request.get_json(force=True) return jsonify(self.input.get_global_causal_policy(data)) self.add_url_rule(global_causal_policy, '/global_causal_policy', methods=["POST"]) def importances(): return jsonify(self.input.importances()) self.add_url_rule(importances, '/importances', methods=["POST"])