def model_support_dmd(self): return GeneralUtils.dmd_supported(self.model, self.test)
def score_value_report( self, model, dmd_test: DMD, labels=None, y_proba: numpy.ndarray = None, y_pred: numpy.ndarray = None) -> [ScoringMetricReport]: ''' :param model: model of interest :param dmd_test: test set :param y_proba: pre-calculated predicted probabilities for test set, if available :param y_pred: pre-calculated models' predictions for test set, if available :return: scoring report ''' score_report = [] model_support_dmd = GeneralUtils.dmd_supported(model, dmd_test) x_test = dmd_test if model_support_dmd else dmd_test.values y_true = dmd_test.target is_classification = GeneralUtils.is_classification(model) confusion_matrix, scatter, classification_report = None, None, None if is_classification: y_proba = y_proba if y_proba is not None else model.predict_proba( x_test) y_pred = y_pred if y_pred is not None else numpy.argmax(y_proba, axis=1) confusion_matrix = ConfusionMatrixReport( y_true=y_true, y_pred=y_pred, labels=labels if labels is not None else unique_labels( y_true, y_pred)) classification_report = SklearnClassificationReport( y_true=y_true, y_pred=y_pred, y_proba=y_proba, labels=labels) for metric in self.metrics: if not metric.ptype == CLASSIFICATION: continue if metric.is_proba: yp = y_proba else: yp = y_pred score = metric.function(y_true, yp) ci_low, ci_high = Metrics.confidence_interval(metric, y_true=y_true, y_pred=y_pred, y_proba=y_proba) score_report.append( ScoringMetricReport(metric_name=metric.name, value=score, ci_low=ci_low, ci_high=ci_high)) else: y_pred = y_pred if y_pred is not None else model.predict(x_test) error_bars = self._calc_error_bars(dmd_test, model) scatter = ScatterReport(y_true=y_true, y_pred=y_pred, error_bars=error_bars) for metric in self.metrics: if not metric.ptype == REGRESSION: continue score = metric.function(y_true, y_pred) ci_low, ci_high = Metrics.confidence_interval(metric, y_true=y_true, y_pred=y_pred) ci_low = GeneralUtils.f5(ci_low) ci_high = GeneralUtils.f5(ci_high) score = GeneralUtils.f5(score) score_report.append( ScoringMetricReport(metric_name=metric.name, value=score, ci_low=ci_low, ci_high=ci_high)) return score_report, confusion_matrix, scatter, classification_report
def fit(self, dmd_test: DMD, **kwargs): self.dmd_supported = GeneralUtils.dmd_supported(model=self.model, dmd=dmd_test) self.fit_uncertainty_model(dmd_test, **kwargs) return self
def sensitivity_analysis(self, model, dmd_test: DMD, metric, dmd_train=None, method=SensitivityTypes.shuffled, raw_scores=False, y_pred=None) -> SensitivityOfFeaturesReport: self.model_support_dmd = GeneralUtils.dmd_supported(model, dmd_test) x = dmd_test if self.model_support_dmd else dmd_test.values y_pred = y_pred or model.predict(x) ytest = dmd_test.target score_function = self.metrics[metric].function if metric in ['auc', 'logloss'] and ytest is not None: base_score = score_function(ytest, model.predict_proba(x)) y_pred = ytest else: base_score = 0 y_pred = y_pred predict_function = model.predict_proba if self.metrics[ metric].is_proba \ else model.predict scores = {} for i, name in enumerate(dmd_test.feature_names): if dmd_test.n_samples > self.max_samples_to_use: rs = numpy.random.RandomState(i) subset = rs.permutation( dmd_test.n_samples)[:self.max_samples_to_use] dmd_test_ = dmd_test.split_by_indices(subset) y_pred_ = y_pred[subset] else: dmd_test_ = dmd_test y_pred_ = y_pred shuffled_x = self.get_shuffled_x( dmd_test_, i, dmd_train=dmd_train, method=method, model_support_dmd=self.model_support_dmd) shuffled_pred = predict_function(shuffled_x) if base_score > 0: scores[name] = 1 - abs( base_score - score_function(y_pred_, shuffled_pred) ) # higher difference - more impact so add 1- in front else: scores[name] = score_function( y_pred_, shuffled_pred) # higher score - less impact if raw_scores: # description = "The raw scores of how each feature affects the model's predictions." return SensitivityOfFeaturesReport( method=method, sensitivities=scores, stats_report=self._sensitivity_stats_report(scores)) # higher score / lower loss means the shuffled feature did less impact if self.metrics[metric].is_loss: impact = scores else: impact = {name: 1 - score for name, score in scores.items()} total_impact = sum([score for score in impact.values()]) impact = { name: float(score / total_impact) for name, score in impact.items() } impact = GeneralUtils.round_values(impact) # description="The impact of each feature on model's predictions. " # "Higher value mean larger impact (0 means no impact at all). " # "Values are normalized to 1.") return SensitivityOfFeaturesReport( method=method, sensitivities=impact, stats_report=self._sensitivity_stats_report(sensitivities=impact))