def _init_metrics(self):
""" Starts up the metrics and statistics watchers. One watcher is created
for each of the learners to be evaluated.
"""
self.mean_eval_measurements = []
self.current_eval_measurements = []
if self._task_type == constants.CLASSIFICATION:
for i in range(self.n_models):
self.mean_eval_measurements.append(
ClassificationMeasurements())
self.current_eval_measurements.append(
WindowClassificationMeasurements(
window_size=self.n_sliding))
elif self._task_type == constants.MULTI_TARGET_CLASSIFICATION:
for i in range(self.n_models):
self.mean_eval_measurements.append(
MultiTargetClassificationMeasurements())
self.current_eval_measurements.append(
WindowMultiTargetClassificationMeasurements(
window_size=self.n_sliding))
elif self._task_type == constants.REGRESSION:
for i in range(self.n_models):
self.mean_eval_measurements.append(RegressionMeasurements())
self.current_eval_measurements.append(
WindowRegressionMeasurements(window_size=self.n_sliding))
elif self._task_type == constants.MULTI_TARGET_REGRESSION:
for i in range(self.n_models):
self.mean_eval_measurements.append(
MultiTargetRegressionMeasurements())
self.current_eval_measurements.append(
WindowMultiTargetRegressionMeasurements(
window_size=self.n_sliding))
# Running time
self.running_time_measurements = []
for i in range(self.n_models):
self.running_time_measurements.append(RunningTimeMeasurements())
# Evaluation data buffer
self._data_dict = {}
for metric in self.metrics:
data_ids = [constants.MEAN, constants.CURRENT]
if metric == constants.TRUE_VS_PREDICTED:
data_ids = [constants.Y_TRUE, constants.Y_PRED]
elif metric == constants.DATA_POINTS:
data_ids = ['X', 'target_values', 'prediction']
elif metric == constants.RUNNING_TIME:
data_ids = [
'training_time', 'testing_time', 'total_running_time'
]
elif metric == constants.MODEL_SIZE:
data_ids = ['model_size']
self._data_dict[metric] = data_ids
self._data_buffer = EvaluationDataBuffer(data_dict=self._data_dict)
def _init_metrics(self):
""" Starts up the metrics and statistics watchers. One watcher is created
for each of the learners to be evaluated.
"""
self.mean_eval_measurements = []
self.current_eval_measurements = []
if self._task_type == constants.CLASSIFICATION:
for i in range(self.n_models):
self.mean_eval_measurements.append(
ClassificationMeasurements())
self.current_eval_measurements.append(
WindowClassificationMeasurements(
window_size=self.n_sliding))
elif self._task_type == constants.MULTI_OUTPUT:
for i in range(self.n_models):
self.mean_eval_measurements.append(MultiOutputMeasurements())
self.current_eval_measurements.append(
WindowMultiOutputMeasurements(window_size=self.n_sliding))
elif self._task_type == constants.REGRESSION:
for i in range(self.n_models):
self.mean_eval_measurements.append(RegressionMeasurements())
self.current_eval_measurements.append(
WindowRegressionMeasurements(window_size=self.n_sliding))
def test_window_classification_measurements():
y_true = np.concatenate((np.ones(85), np.zeros(10), np.ones(5)))
y_pred = np.concatenate((np.ones(90), np.zeros(10)))
measurements = WindowClassificationMeasurements(window_size=20)
for i in range(len(y_true)):
measurements.add_result(y_true[i], y_pred[i])
expected_acc = 10 / 20
assert expected_acc == measurements.get_accuracy()
expected_incorrectly_classified_ratio = 1 - expected_acc
assert expected_incorrectly_classified_ratio == measurements.get_incorrectly_classified_ratio(
)
expected_kappa = 0.0
assert np.isclose(expected_kappa, measurements.get_kappa())
expected_kappa_m = 0.3333333333333333
assert np.isclose(expected_kappa_m, measurements.get_kappa_m())
expected_kappa_t = -4.0
assert np.isclose(expected_kappa_t, measurements.get_kappa_t())
expected_precision = 5 / (5 + 5)
assert np.isclose(expected_precision, measurements.get_precision())
expected_recall = 5 / (5 + 5)
assert np.isclose(expected_recall, measurements.get_recall())
expected_f1_score = 2 * ((expected_precision * expected_recall) /
(expected_precision + expected_recall))
assert np.isclose(expected_f1_score, measurements.get_f1_score())
expected_g_mean = np.sqrt((5 / (5 + 5)) * expected_recall)
assert np.isclose(expected_g_mean, measurements.get_g_mean())
expected_info = 'WindowClassificationMeasurements: - sample_count: 20 - window_size: 20 ' \
'- accuracy: 0.500000 - kappa: 0.000000 - kappa_t: -4.000000 ' \
'- kappa_m: 0.333333 - f1-score: 0.500000 - precision: 0.500000 ' \
'- recall: 0.500000 - g-mean: 0.500000 - majority_class: 0'
assert expected_info == measurements.get_info()
expected_last = (1.0, 0.0)
assert expected_last == measurements.get_last()
expected_majority_class = 0
assert expected_majority_class == measurements.get_majority_class()
measurements.reset()
assert measurements.sample_count == 0
def test_window_classification_measurements():
y_true = np.ones(100)
y_pred = np.concatenate((np.ones(90), np.zeros(10)))
measurements = WindowClassificationMeasurements(window_size=20)
for i in range(len(y_true)):
measurements.add_result(y_true[i], y_pred[i])
expected_acc = .5
assert expected_acc == measurements.get_accuracy()
expected_incorrectly_classified_ratio = 1 - expected_acc
assert expected_incorrectly_classified_ratio == measurements.get_incorrectly_classified_ratio()
expected_kappa = 0.0
assert expected_kappa == measurements.get_kappa()
expected_kappa_m = 0.5
assert expected_kappa_m == measurements.get_kappa_m()
expected_kappa_t = 1
assert expected_kappa_t == measurements.get_kappa_t()
expected_info = 'WindowClassificationMeasurements: - sample_count: 20 - window_size: 20 - accuracy: 0.500000 ' \
'- kappa: 0.000000 - kappa_t: 1.000000 - kappa_m: 0.500000 - majority_class: 0'
assert expected_info == measurements.get_info()
expected_last = (1.0, 0.0)
assert expected_last == measurements.get_last()
expected_majority_class = 0
assert expected_majority_class == measurements.get_majority_class()
measurements.reset()
assert measurements.sample_count == 0
assert measurements.get_class_type() == 'measurement'