def scorer(model: Model, data: Table) -> float:
is_cls = data.domain.has_discrete_class
if not needs_preprocessing and hasattr(model, "skl_model"):
pred = model.skl_model.predict(data.X)
if is_cls:
prob = model.skl_model.predict_proba(data.X)
# TODO - unify model.predict() output for all Models
# elif not needs_preprocessing:
# pred = model.predict(data.X)
# if is_cls:
# assert isinstance(pred, tuple)
# pred, prob = pred
else:
if is_cls:
pred, prob = model(data, ret=Model.ValueProbs)
else:
pred = model(data, ret=Model.Value)
results = Results(data, domain=data.domain, actual=data.Y,
predicted=pred.reshape((1, len(data))))
if is_cls:
results.probabilities = prob.reshape((1,) + prob.shape)
if isinstance(score, TargetScore):
return score.compute_score(results, average="weighted")[0]
else:
return score.compute_score(results)[0]
def compute_auc(self, actual, predicted):
predicted = np.array(predicted).reshape(1, -1)
probabilities = np.zeros((1, predicted.shape[1], 2))
probabilities[0,:,1] = predicted[0]
probabilities[0,:,0] = 1 - predicted[0]
results = Results(
nmethods=1, domain=Domain([], [DiscreteVariable(values='01')]),
actual=actual, predicted=predicted)
results.probabilities = probabilities
return AUC(results)[0]
def compute_auc(self, actual, predicted):
predicted = np.array(predicted).reshape(1, -1)
probabilities = np.zeros((1, predicted.shape[1], 2))
probabilities[0, :, 1] = predicted[0]
probabilities[0, :, 0] = 1 - predicted[0]
results = Results(
nmethods=1, domain=Domain([], [DiscreteVariable(values='01')]),
actual=actual, predicted=predicted)
results.probabilities = probabilities
return AUC(results)[0]
def _call_predictors(self):
if not self.data:
return
if self.class_var:
domain = self.data.domain
classless_data = self.data.transform(
Domain(domain.attributes, None, domain.metas))
else:
classless_data = self.data
for inputid, slot in self.predictors.items():
if isinstance(slot.results, Results):
continue
predictor = slot.predictor
try:
if predictor.domain.class_var.is_discrete:
pred, prob = predictor(classless_data, Model.ValueProbs)
else:
pred = predictor(classless_data, Model.Value)
prob = numpy.zeros((len(pred), 0))
except (ValueError, DomainTransformationError) as err:
self.predictors[inputid] = \
slot._replace(results=f"{predictor.name}: {err}")
continue
results = Results()
results.data = self.data
results.domain = self.data.domain
results.row_indices = numpy.arange(len(self.data))
results.folds = (Ellipsis, )
results.actual = self.data.Y
results.unmapped_probabilities = prob
results.unmapped_predicted = pred
results.probabilities = results.predicted = None
self.predictors[inputid] = slot._replace(results=results)
target = predictor.domain.class_var
if target != self.class_var:
continue
if target is not self.class_var and target.is_discrete:
backmappers, n_values = predictor.get_backmappers(self.data)
prob = predictor.backmap_probs(prob, n_values, backmappers)
pred = predictor.backmap_value(pred, prob, n_values,
backmappers)
results.predicted = pred.reshape((1, len(self.data)))
results.probabilities = prob.reshape((1, ) + prob.shape)
def _commit_evaluation_results(self):
slots = [p for p in self._non_errored_predictors()
if p.results.predicted is not None]
if not slots:
self.Outputs.evaluation_results.send(None)
return
nanmask = numpy.isnan(self.data.get_column_view(self.class_var)[0])
data = self.data[~nanmask]
results = Results(data, store_data=True)
results.folds = None
results.row_indices = numpy.arange(len(data))
results.actual = data.Y.ravel()
results.predicted = numpy.vstack(
tuple(p.results.predicted[0][~nanmask] for p in slots))
if self.class_var and self.class_var.is_discrete:
results.probabilities = numpy.array(
[p.results.probabilities[0][~nanmask] for p in slots])
results.learner_names = [p.name for p in slots]
self.Outputs.evaluation_results.send(results)
