def _init_metrics(self):
""" Starts up the metrics and statistics watchers. One watcher is created
for each of the learners to be evaluated.
"""
self.mean_eval_measurements = []
self.current_eval_measurements = []
if self._task_type == constants.CLASSIFICATION:
for i in range(self.n_models):
self.mean_eval_measurements.append(
ClassificationMeasurements())
self.current_eval_measurements.append(
WindowClassificationMeasurements(
window_size=self.n_sliding))
elif self._task_type == constants.MULTI_TARGET_CLASSIFICATION:
for i in range(self.n_models):
self.mean_eval_measurements.append(
MultiTargetClassificationMeasurements())
self.current_eval_measurements.append(
WindowMultiTargetClassificationMeasurements(
window_size=self.n_sliding))
elif self._task_type == constants.REGRESSION:
for i in range(self.n_models):
self.mean_eval_measurements.append(RegressionMeasurements())
self.current_eval_measurements.append(
WindowRegressionMeasurements(window_size=self.n_sliding))
elif self._task_type == constants.MULTI_TARGET_REGRESSION:
for i in range(self.n_models):
self.mean_eval_measurements.append(
MultiTargetRegressionMeasurements())
self.current_eval_measurements.append(
WindowMultiTargetRegressionMeasurements(
window_size=self.n_sliding))
# Running time
self.running_time_measurements = []
for i in range(self.n_models):
self.running_time_measurements.append(RunningTimeMeasurements())
# Evaluation data buffer
self._data_dict = {}
for metric in self.metrics:
data_ids = [constants.MEAN, constants.CURRENT]
if metric == constants.TRUE_VS_PREDICTED:
data_ids = [constants.Y_TRUE, constants.Y_PRED]
elif metric == constants.DATA_POINTS:
data_ids = ['X', 'target_values', 'prediction']
elif metric == constants.RUNNING_TIME:
data_ids = [
'training_time', 'testing_time', 'total_running_time'
]
elif metric == constants.MODEL_SIZE:
data_ids = ['model_size']
self._data_dict[metric] = data_ids
self._data_buffer = EvaluationDataBuffer(data_dict=self._data_dict)
def test_multi_target_classification_measurements():
y_0 = np.ones(100)
y_1 = np.concatenate((np.ones(90), np.zeros(10)))
y_2 = np.concatenate((np.ones(85), np.zeros(10), np.ones(5)))
y_true = np.ones((100, 3))
y_pred = np.vstack((y_0, y_1, y_2)).T
measurements = MultiTargetClassificationMeasurements()
for i in range(len(y_true)):
measurements.add_result(y_true[i], y_pred[i])
expected_acc = 0.85
assert np.isclose(expected_acc, measurements.get_exact_match())
expected_hamming_score = 1 - 0.06666666666666667
assert np.isclose(expected_hamming_score, measurements.get_hamming_score())
expected_hamming_loss = 0.06666666666666667
assert np.isclose(expected_hamming_loss, measurements.get_hamming_loss())
expected_jaccard_index = 0.9333333333333332
assert np.isclose(expected_jaccard_index, measurements.get_j_index())
expected_total_sum = 300
assert expected_total_sum == measurements.get_total_sum()
expected_info = 'MultiTargetClassificationMeasurements: - sample_count: 100 - hamming_loss: 0.066667 - ' \
'hamming_score: 0.933333 - exact_match: 0.850000 - j_index: 0.933333'
assert expected_info == measurements.get_info()
expected_last_true = (1.0, 1.0, 1.0)
expected_last_pred = (1.0, 0.0, 1.0)
assert np.alltrue(expected_last_true == measurements.get_last()[0])
assert np.alltrue(expected_last_pred == measurements.get_last()[1])
measurements.reset()
assert measurements.sample_count == 0