def scorer(model: Model, data: Table) -> float: is_cls = data.domain.has_discrete_class if not needs_preprocessing and hasattr(model, "skl_model"): pred = model.skl_model.predict(data.X) if is_cls: prob = model.skl_model.predict_proba(data.X) # TODO - unify model.predict() output for all Models # elif not needs_preprocessing: # pred = model.predict(data.X) # if is_cls: # assert isinstance(pred, tuple) # pred, prob = pred else: if is_cls: pred, prob = model(data, ret=Model.ValueProbs) else: pred = model(data, ret=Model.Value) results = Results(data, domain=data.domain, actual=data.Y, predicted=pred.reshape((1, len(data)))) if is_cls: results.probabilities = prob.reshape((1,) + prob.shape) if isinstance(score, TargetScore): return score.compute_score(results, average="weighted")[0] else: return score.compute_score(results)[0]
def compute_auc(self, actual, predicted): predicted = np.array(predicted).reshape(1, -1) probabilities = np.zeros((1, predicted.shape[1], 2)) probabilities[0,:,1] = predicted[0] probabilities[0,:,0] = 1 - predicted[0] results = Results( nmethods=1, domain=Domain([], [DiscreteVariable(values='01')]), actual=actual, predicted=predicted) results.probabilities = probabilities return AUC(results)[0]
def compute_auc(self, actual, predicted): predicted = np.array(predicted).reshape(1, -1) probabilities = np.zeros((1, predicted.shape[1], 2)) probabilities[0, :, 1] = predicted[0] probabilities[0, :, 0] = 1 - predicted[0] results = Results( nmethods=1, domain=Domain([], [DiscreteVariable(values='01')]), actual=actual, predicted=predicted) results.probabilities = probabilities return AUC(results)[0]
def _call_predictors(self): if not self.data: return if self.class_var: domain = self.data.domain classless_data = self.data.transform( Domain(domain.attributes, None, domain.metas)) else: classless_data = self.data for inputid, slot in self.predictors.items(): if isinstance(slot.results, Results): continue predictor = slot.predictor try: if predictor.domain.class_var.is_discrete: pred, prob = predictor(classless_data, Model.ValueProbs) else: pred = predictor(classless_data, Model.Value) prob = numpy.zeros((len(pred), 0)) except (ValueError, DomainTransformationError) as err: self.predictors[inputid] = \ slot._replace(results=f"{predictor.name}: {err}") continue results = Results() results.data = self.data results.domain = self.data.domain results.row_indices = numpy.arange(len(self.data)) results.folds = (Ellipsis, ) results.actual = self.data.Y results.unmapped_probabilities = prob results.unmapped_predicted = pred results.probabilities = results.predicted = None self.predictors[inputid] = slot._replace(results=results) target = predictor.domain.class_var if target != self.class_var: continue if target is not self.class_var and target.is_discrete: backmappers, n_values = predictor.get_backmappers(self.data) prob = predictor.backmap_probs(prob, n_values, backmappers) pred = predictor.backmap_value(pred, prob, n_values, backmappers) results.predicted = pred.reshape((1, len(self.data))) results.probabilities = prob.reshape((1, ) + prob.shape)
def _commit_evaluation_results(self): slots = [p for p in self._non_errored_predictors() if p.results.predicted is not None] if not slots: self.Outputs.evaluation_results.send(None) return nanmask = numpy.isnan(self.data.get_column_view(self.class_var)[0]) data = self.data[~nanmask] results = Results(data, store_data=True) results.folds = None results.row_indices = numpy.arange(len(data)) results.actual = data.Y.ravel() results.predicted = numpy.vstack( tuple(p.results.predicted[0][~nanmask] for p in slots)) if self.class_var and self.class_var.is_discrete: results.probabilities = numpy.array( [p.results.probabilities[0][~nanmask] for p in slots]) results.learner_names = [p.name for p in slots] self.Outputs.evaluation_results.send(results)