def test_empty_results(self):
"""Test on empty results."""
res = Results(data=self.iris[:0], store_data=True)
res.row_indices = np.array([], dtype=int)
res.actual = np.array([])
res.predicted = np.array([[]])
res.probabilities = np.zeros((1, 0, 3))
self.send_signal(self.widget.Inputs.evaluation_results, res)
self.widget.select_correct()
self.widget.select_wrong()
def test_curves_from_results_nans(self, init):
res = Results()
ytrue, probs = self.data.T
ytrue[0] = np.nan
probs[-1] = np.nan
res.actual = ytrue.astype(float)
res.probabilities = np.vstack((1 - probs, probs)).T.reshape(1, -1, 2)
Curves.from_results(res)
cytrue, cprobs = init.call_args[0]
np.testing.assert_equal(cytrue, ytrue[1:-1])
np.testing.assert_equal(cprobs, probs[1:-1])
def test_nan_results(self):
"""Test on results with nan values in actual/predicted"""
res = Results(data=self.iris, nmethods=2, store_data=True)
res.row_indices = np.array([0, 50, 100], dtype=int)
res.actual = np.array([0., np.nan, 2.])
res.predicted = np.array([[np.nan, 1, 2],
[np.nan, np.nan, np.nan]])
res.probabilities = np.zeros((1, 3, 3))
self.send_signal(self.widget.Inputs.evaluation_results, res)
self.assertTrue(self.widget.Error.invalid_values.is_shown())
self.send_signal(self.widget.Inputs.evaluation_results, None)
self.assertFalse(self.widget.Error.invalid_values.is_shown())
def test_curves_from_results(self, init):
res = Results()
ytrue, probs = self.data.T
res.actual = ytrue.astype(float)
res.probabilities = np.vstack((1 - probs, probs)).T.reshape(1, -1, 2)
Curves.from_results(res)
cytrue, cprobs = init.call_args[0]
np.testing.assert_equal(cytrue, ytrue)
np.testing.assert_equal(cprobs, probs)
Curves.from_results(res, target_class=0)
cytrue, cprobs = init.call_args[0]
np.testing.assert_equal(cytrue, 1 - ytrue)
np.testing.assert_equal(cprobs, 1 - probs)
res.actual = ytrue.astype(float)
res.probabilities = np.random.random((2, 19, 2))
res.probabilities[1] = np.vstack((1 - probs, probs)).T
Curves.from_results(res, model_index=1)
cytrue, cprobs = init.call_args[0]
np.testing.assert_equal(cytrue, ytrue)
np.testing.assert_equal(cprobs, probs)
self.assertRaises(ValueError, Curves.from_results, res)
ytrue[ytrue == 0] = 2 * (np.arange(10) % 2)
res.actual = ytrue.astype(float)
res.probabilities = np.random.random((2, 19, 3))
res.probabilities[1] = np.vstack(
((1 - probs) / 3, probs, (1 - probs) * 2 / 3)).T
Curves.from_results(res, model_index=1, target_class=1)
cytrue, cprobs = init.call_args[0]
np.testing.assert_equal(cytrue, ytrue == 1)
np.testing.assert_equal(cprobs, probs)
Curves.from_results(res, model_index=1, target_class=0)
cytrue, cprobs = init.call_args[0]
np.testing.assert_equal(cytrue, ytrue == 0)
np.testing.assert_equal(cprobs, (1 - probs) / 3)
Curves.from_results(res, model_index=1, target_class=2)
cytrue, cprobs = init.call_args[0]
np.testing.assert_equal(cytrue, ytrue == 2)
np.testing.assert_equal(cprobs, (1 - probs) * 2 / 3)
self.assertRaises(ValueError, Curves.from_results, res, model_index=1)
def _update(self):
assert self.data is not None
# collect all learners for which results have not yet been computed
need_update = [(id, learner) for id, learner in self.learners.items()
if self.results[id] is None]
if not need_update:
return
learners = [learner for _, learner in need_update]
self.progressBarInit()
if self.testdata is None:
# compute the learning curve result for all learners in one go
results = learning_curve(
learners,
self.data,
folds=self.folds,
proportions=self.curvePoints,
callback=lambda value: self.progressBarSet(100 * value))
else:
results = learning_curve_with_test_data(
learners,
self.data,
self.testdata,
times=self.folds,
proportions=self.curvePoints,
callback=lambda value: self.progressBarSet(100 * value))
self.progressBarFinished()
# split the combined result into per learner/model results
results = [
list(Results.split_by_model(p_results)) for p_results in results
]
for i, (id, learner) in enumerate(need_update):
self.results[id] = [p_results[i] for p_results in results]
def _update(self):
assert self.data is not None
# collect all learners for which results have not yet been computed
need_update = [(i, item) for (i, item) in enumerate(self.learners)
if item.results is None]
if not need_update:
return
learners = [item.learner for _, item in need_update]
if self.testdata is None:
# compute the learning curve result for all learners in one go
results = learning_curve(
learners,
self.data,
folds=self.folds,
proportions=self.curvePoints,
)
else:
results = learning_curve_with_test_data(
learners,
self.data,
self.testdata,
times=self.folds,
proportions=self.curvePoints,
)
# split the combined result into per learner/model results
results = [
list(Results.split_by_model(p_results)) for p_results in results
]
for i, (_, item) in enumerate(need_update):
item.results = [p_results[i] for p_results in results]
def _update(self):
assert self.data is not None
# collect all learners for which results have not yet been computed
need_update = [(id, learner) for id, learner in self.learners.items()
if self.results[id] is None]
if not need_update:
return
learners = [learner for _, learner in need_update]
if self.testdata is None:
# compute the learning curve result for all learners in one go
results = learning_curve(
learners, self.data, folds=self.folds,
proportions=self.curvePoints,
)
else:
results = learning_curve_with_test_data(
learners, self.data, self.testdata, times=self.folds,
proportions=self.curvePoints,
)
# split the combined result into per learner/model results
results = [list(Results.split_by_model(p_results)) for p_results in results]
for i, (id, learner) in enumerate(need_update):
self.results[id] = [p_results[i] for p_results in results]
def _update(self):
assert self.data is not None
# collect all learners for which results have not yet been computed
need_update = [(id, learner) for id, learner in self.learners.items()
if self.results[id] is None]
if not need_update: #啥意思？
return
learners = [learner for _, learner in need_update]
# compute the learning curve result for all learners in one go
results = learning_curve(
learners,
self.data,
folds=self.folds,
proportions=self.curvePoints,
) #这是一个元组
# split the combined result into per learner/model results
results = [
list(Results.split_by_model(p_results)) for p_results in results
]
for i, (id, learner) in enumerate(need_update):
self.results[id] = [p_results[i] for p_results in results]
def _create_results(): # pragma: no cover
probs1 = [0.984, 0.907, 0.881, 0.865, 0.815, 0.741, 0.735, 0.635,
0.582, 0.554, 0.413, 0.317, 0.287, 0.225, 0.216, 0.183]
probs = np.array([[[1 - x, x] for x in probs1]])
preds = (probs > 0.5).astype(float)
return Results(
data=Orange.data.Table("heart_disease")[:16],
row_indices=np.arange(16),
actual=np.array(list(map(int, "1100111001001000"))),
probabilities=probs, predicted=preds
)
def get_fold(self, fold):
results = Results()
results.data = self.data
if self.folds is None:
raise ValueError("This 'Results' instance does not have folds.")
if self.models is not None:
results.models = self.models[fold]
results.row_indices = self.row_indices
results.actual = self.actual
results.predicted = self.predicted[:, fold, :]
results.domain = self.domain
return results
def test_empty_results(self):
"""Test on empty results."""
res = Results(data=self.iris[:0], store_data=True)
res.row_indices = np.array([], dtype=int)
res.actual = np.array([])
res.predicted = np.array([[]])
res.probabilities = np.zeros((1, 0, 3))
self.send_signal(self.widget.Inputs.evaluation_results, res)
self.widget.select_correct()
self.widget.select_wrong()
def _task_finished(self, f):
"""
Parameters
----------
f : Future
The future instance holding the result of learner evaluation.
"""
assert self.thread() is QThread.currentThread()
assert self._task is not None
assert self._task.future is f
assert f.done()
self._task = None
self.progressBarFinished()
try:
results = f.result() # type: List[Results]
except Exception as ex:
# Log the exception with a traceback
log = logging.getLogger()
log.exception(__name__, exc_info=True)
self.error("Exception occurred during evaluation: {!r}".format(ex))
# clear all results
for key in self.results.keys():
self.results[key] = None
else:
# split the combined result into per learner/model results ...
results = [
list(Results.split_by_model(p_results))
for p_results in results
] # type: List[List[Results]]
assert all(len(r.learners) == 1 for r1 in results for r in r1)
assert len(results) == len(self.curvePoints)
learners = [r.learners[0] for r in results[0]]
learner_id = {
learner: id_
for id_, learner in self.learners.items()
}
# ... and update self.results
for i, learner in enumerate(learners):
id_ = learner_id[learner]
self.results[id_] = [p_results[i] for p_results in results]
# [end-snippet-9]
# update the display
self._update_curve_points()
self._update_table()
def test_nan_results(self):
"""Test on results with nan values in actual/predicted"""
res = Results(data=self.iris, nmethods=2, store_data=True)
res.row_indices = np.array([0, 50, 100], dtype=int)
res.actual = np.array([0., np.nan, 2.])
res.predicted = np.array([[np.nan, 1, 2], [np.nan, np.nan, np.nan]])
res.probabilities = np.zeros((1, 3, 3))
self.send_signal(self.widget.Inputs.evaluation_results, res)
self.assertTrue(self.widget.Error.invalid_values.is_shown())
self.send_signal(self.widget.Inputs.evaluation_results, None)
self.assertFalse(self.widget.Error.invalid_values.is_shown())
def _task_finished(self, f):
"""
Parameters
----------
f : Future
The future instance holding the result of learner evaluation.
"""
assert self.thread() is QThread.currentThread()
assert self._task is not None
assert self._task.future is f
assert f.done()
self._task = None
self.progressBarFinished()
try:
results = f.result() # type: List[Results]
except Exception as ex:
# Log the exception with a traceback
log = logging.getLogger()
log.exception(__name__, exc_info=True)
self.error("Exception occurred during evaluation: {!r}"
.format(ex))
# clear all results
for key in self.results.keys():
self.results[key] = None
else:
# split the combined result into per learner/model results ...
results = [list(Results.split_by_model(p_results))
for p_results in results] # type: List[List[Results]]
assert all(len(r.learners) == 1 for r1 in results for r in r1)
assert len(results) == len(self.curvePoints)
learners = [r.learners[0] for r in results[0]]
learner_id = {learner: id_ for id_, learner in self.learners.items()}
# ... and update self.results
for i, learner in enumerate(learners):
id_ = learner_id[learner]
self.results[id_] = [p_results[i] for p_results in results]
# [end-snippet-9]
# update the display
self._update_curve_points()
self._update_table()
def _update(self):
assert self.data is not None
# collect all learners for which results have not yet been computed
need_update = [(id, learner) for id, learner in self.learners.items()
if self.results[id] is None]
if not need_update:
return
learners = [learner for _, learner in need_update]
self.progressBarInit()
# compute the learning curve result for all learners in one go
results = learning_curve(
learners, self.data, folds=self.folds,
proportions=self.curvePoints,
callback=lambda value: self.progressBarSet(100 * value)
)
self.progressBarFinished()
# split the combined result into per learner/model results
results = [list(Results.split_by_model(p_results))
for p_results in results]
for i, (id, learner) in enumerate(need_update):
self.results[id] = [p_results[i] for p_results in results]