def results_add_by_model(x, y):
def is_empty(res):
return (getattr(res, "models", None) is None
and getattr(res, "row_indices", None) is None)
if is_empty(x):
return y
elif is_empty(y):
return x
assert (x.row_indices == y.row_indices).all()
assert (x.actual == y.actual).all()
res = testing.Results()
res.data = x.data
res.row_indices = x.row_indices
res.folds = x.folds
res.actual = x.actual
res.predicted = numpy.vstack((x.predicted, y.predicted))
if x.probabilities is not None and y.probabilities is not None:
res.probabilities = numpy.vstack((x.probabilities, y.probabilities))
if x.models is not None:
res.models = [xm + ym for xm, ym in zip(x.models, y.models)]
return res
def test_init(self):
res = testing.Results(nmethods=2, nrows=100)
res.actual[:50] = 0
res.actual[50:] = 1
res.predicted = np.vstack((res.actual, res.actual))
np.testing.assert_almost_equal(scoring.CA(res), [1, 1])
res.predicted[0][0] = 1
np.testing.assert_almost_equal(scoring.CA(res), [0.99, 1])
res.predicted[1] = 1 - res.predicted[1]
np.testing.assert_almost_equal(scoring.CA(res), [0.99, 0])
def split_by_model(results):
"""
Split evaluation results by models
"""
data = results.data
nmethods = len(results.predicted)
for i in range(nmethods):
res = testing.Results()
res.data = data
res.row_indices = results.row_indices
res.actual = results.actual
res.predicted = results.predicted[(i,), :]
if results.probabilities is not None:
res.probabilities = results.probabilities[(i,), :, :]
if results.models:
res.models = [mf[i] for mf in results.models]
if results.folds:
res.folds = results.folds
yield res
def results_merge(results):
return functools.reduce(results_add_by_model, results, testing.Results())
def commit(self):
if self.data is None or not self.predictors:
self.send("Predictions", None)
self.send("Evaluation Results", None)
return
predictor = next(iter(self.predictors.values())).predictor
class_var = predictor.domain.class_var
classification = is_discrete(class_var)
newattrs = []
newcolumns = []
slots = list(self.predictors.values())
if classification:
if self.show_class:
mc = [
Orange.data.DiscreteVariable(name=p.name,
values=class_var.values)
for p in slots
]
newattrs.extend(mc)
newcolumns.extend(p.results[0].reshape((-1, 1)) for p in slots)
if self.show_probabilities:
for p in slots:
m = [
Orange.data.ContinuousVariable(name="%s(%s)" %
(p.name, value))
for value in class_var.values
]
newattrs.extend(m)
newcolumns.extend(p.results[1] for p in slots)
else:
# regression
mc = [
Orange.data.ContinuousVariable(name=p.name)
for p in self.predictors.values()
]
newattrs.extend(mc)
newcolumns.extend(p.results[0].reshape((-1, 1)) for p in slots)
domain = Orange.data.Domain(self.data.domain.attributes,
self.data.domain.class_var,
metas=tuple(newattrs))
if newcolumns:
newcolumns = [numpy.atleast_2d(cols) for cols in newcolumns]
newcolumns = numpy.hstack(tuple(newcolumns))
else:
newcolumns = None
predictions = Orange.data.Table.from_numpy(domain,
self.data.X,
self.data.Y,
metas=newcolumns)
predictions.name = self.data.name
results = None
if self.data.domain.class_var == class_var:
N = len(self.data)
results = testing.Results(self.data, store_data=True)
results.folds = None
results.row_indices = numpy.arange(N)
results.actual = self.data.Y.ravel()
results.predicted = numpy.vstack(tuple(p.results[0]
for p in slots))
if classification:
results.probabilities = numpy.array(
[p.results[1] for p in slots])
results.learner_names = [pname(p.predictor) for p in slots]
self.send("Predictions", predictions)
self.send("Evaluation Results", results)