def _init_metrics(self): """ Starts up the metrics and statistics watchers. One watcher is created for each of the learners to be evaluated. """ self.mean_eval_measurements = [] self.current_eval_measurements = [] if self._task_type == constants.CLASSIFICATION: for i in range(self.n_models): self.mean_eval_measurements.append( ClassificationMeasurements()) self.current_eval_measurements.append( WindowClassificationMeasurements( window_size=self.n_sliding)) elif self._task_type == constants.MULTI_TARGET_CLASSIFICATION: for i in range(self.n_models): self.mean_eval_measurements.append( MultiTargetClassificationMeasurements()) self.current_eval_measurements.append( WindowMultiTargetClassificationMeasurements( window_size=self.n_sliding)) elif self._task_type == constants.REGRESSION: for i in range(self.n_models): self.mean_eval_measurements.append(RegressionMeasurements()) self.current_eval_measurements.append( WindowRegressionMeasurements(window_size=self.n_sliding)) elif self._task_type == constants.MULTI_TARGET_REGRESSION: for i in range(self.n_models): self.mean_eval_measurements.append( MultiTargetRegressionMeasurements()) self.current_eval_measurements.append( WindowMultiTargetRegressionMeasurements( window_size=self.n_sliding)) # Running time self.running_time_measurements = [] for i in range(self.n_models): self.running_time_measurements.append(RunningTimeMeasurements()) # Evaluation data buffer self._data_dict = {} for metric in self.metrics: data_ids = [constants.MEAN, constants.CURRENT] if metric == constants.TRUE_VS_PREDICTED: data_ids = [constants.Y_TRUE, constants.Y_PRED] elif metric == constants.DATA_POINTS: data_ids = ['X', 'target_values', 'prediction'] elif metric == constants.RUNNING_TIME: data_ids = [ 'training_time', 'testing_time', 'total_running_time' ] elif metric == constants.MODEL_SIZE: data_ids = ['model_size'] self._data_dict[metric] = data_ids self._data_buffer = EvaluationDataBuffer(data_dict=self._data_dict)
def test_multi_target_regression_measurements(): y_true = np.zeros((100, 3)) y_pred = np.zeros((100, 3)) for t in range(3): y_true[:, t] = np.sin(range(100)) y_pred[:, t] = np.sin(range(100)) + (t + 1) * .05 measurements = MultiTargetRegressionMeasurements() for i in range(100): measurements.add_result(y_true[i], y_pred[i]) expected_amse = 0.011666666666666664 assert np.isclose(expected_amse, measurements.get_average_mean_square_error()) expected_aae = 0.09999999999999999 assert np.isclose(expected_aae, measurements.get_average_absolute_error()) expected_armse = 0.09999999999999999 assert np.isclose(expected_armse, measurements.get_average_root_mean_square_error()) expected_info = 'MultiTargetRegressionMeasurements: sample_count: 100 - ' \ 'average_mean_square_error: {} - ' \ 'average_mean_absolute_error: {} - ' \ 'average_root_mean_square_error: {}'.format( str(measurements.get_average_mean_square_error()), str(measurements.get_average_absolute_error()), str(measurements.get_average_root_mean_square_error()) ) assert expected_info == measurements.get_info() expected_last = (np.array([-0.99920683, -0.99920683, -0.99920683]), np.array([-0.94920683, -0.89920683, -0.84920683])) for exp, obs in zip(expected_last, measurements.get_last()): assert np.isclose(exp, obs).all() measurements.reset() assert measurements.sample_count == 0