def reset(self):
self.total_square_error = 0.0
self.average_error = 0.0
self.last_true_label = None
self.last_prediction = None
self.total_square_error_correction = FastBuffer(self.window_size)
self.average_error_correction = FastBuffer(self.window_size)
def __init__(self, window_size=200):
super().__init__()
self.total_square_error = 0.0
self.average_error = 0.0
self.last_true_label = None
self.last_prediction = None
self.total_square_error_correction = FastBuffer(window_size)
self.average_error_correction = FastBuffer(window_size)
self.window_size = window_size
def reset(self):
if self.targets is not None:
self.n_targets = len(self.targets)
else:
self.n_targets = 2
self.true_labels = FastBuffer(self.window_size)
self.predictions = FastBuffer(self.window_size)
self.temp = 0
self.last_prediction = None
self.last_true_label = None
self.last_sample = None
self.majority_classifier = 0
self.correct_no_change = 0
self.confusion_matrix.restart(self.n_targets)
self.majority_classifier_correction = FastBuffer(self.window_size)
self.correct_no_change_correction = FastBuffer(self.window_size)
self.rranks = FastBuffer(self.window_size)
def __init__(self, targets=None, dtype=np.int64, window_size=200):
super().__init__()
if targets is not None:
self.n_targets = len(targets)
else:
self.n_targets = 2
self.confusion_matrix = ConfusionMatrix(self.n_targets, dtype)
self.last_class = None
self.rranks = FastBuffer(window_size)
self.targets = targets
self.window_size = window_size
self.true_labels = FastBuffer(window_size)
self.predictions = FastBuffer(window_size)
self.temp = 0
self.last_prediction = None
self.last_true_label = None
self.last_sample = None
self.majority_classifier = 0
self.correct_no_change = 0
self.majority_classifier_correction = FastBuffer(window_size)
self.correct_no_change_correction = FastBuffer(window_size)
class WindowRegressionMeasurements(object):
""" This class is used to keep updated statistics over a regression
learner in a regression problem context inside a fixed sized window.
It uses FastBuffer objects to simulate the fixed sized windows.
It will keep track of partial metrics, that can be provided at
any moment. The relevant metrics kept by an instance of this class
are: MSE (mean square error) and MAE (mean absolute error).
"""
def __init__(self, window_size=200):
super().__init__()
self.total_square_error = 0.0
self.average_error = 0.0
self.last_true_label = None
self.last_prediction = None
self.total_square_error_correction = FastBuffer(window_size)
self.average_error_correction = FastBuffer(window_size)
self.window_size = window_size
def reset(self):
self.total_square_error = 0.0
self.average_error = 0.0
self.last_true_label = None
self.last_prediction = None
self.total_square_error_correction = FastBuffer(self.window_size)
self.average_error_correction = FastBuffer(self.window_size)
def add_result(self, y_true, y_pred):
""" Use the true value and the prediction to update the statistics.
Parameters
----------
y_true: float
The true value.
y_pred: float
The predicted value.
"""
self.last_true_label = y_true
self.last_prediction = y_pred
self.total_square_error += (y_true - y_pred) * (y_true - y_pred)
self.average_error += np.absolute(y_true - y_pred)
old_square = self.total_square_error_correction.add_element(
np.array([-1 * ((y_true - y_pred) * (y_true - y_pred))]))
old_average = self.average_error_correction.add_element(np.array([-1 * (np.absolute(y_true - y_pred))]))
if (old_square is not None) and (old_average is not None):
self.total_square_error += old_square[0]
self.average_error += old_average[0]
def get_mean_square_error(self):
""" Computes the window/current mean square error.
Returns
-------
float
The window/current mean square error.
"""
if self.sample_count == 0:
return 0.0
else:
return self.total_square_error / self.sample_count
def get_average_error(self):
""" Computes the window/current average error.
Returns
-------
float
The window/current average error.
"""
if self.sample_count == 0:
return 0.0
else:
return self.average_error / self.sample_count
def get_last(self):
return self.last_true_label, self.last_prediction
@property
def sample_count(self):
return self.total_square_error_correction.get_current_size()
def get_info(self):
return '{}:'.format(type(self).__name__) + \
' - sample_count: {}'.format(self.sample_count) + \
' - mean_square_error: {:.6f}'.format(self.get_mean_square_error()) + \
' - mean_absolute_error: {:.6f}'.format(self.get_average_error())
class WindowClassificationMeasurements(object):
""" This class will maintain a fixed sized window of the newest information
about one classifier. It can provide, as requested, any of the relevant
current metrics about the classifier, measured inside the window.
To keep track of statistics inside a window, the class will use a
ConfusionMatrix object, alongside FastBuffers, to simulate fixed sized
windows of the important classifier's attributes.
Its functionality is somewhat similar to those of the
ClassificationMeasurements class. The difference is that the statistics
kept by this class are local, or partial, while the statistics kept by
the ClassificationMeasurements class are global.
Parameters
----------
targets: list
A list containing the possible labels.
dtype: data type (Default: numpy.int64)
The data type of the existing labels.
window_size: int (Default: 200)
The width of the window. Determines how many samples the object
can see.
Examples
--------
"""
def __init__(self, targets=None, dtype=np.int64, window_size=200):
super().__init__()
if targets is not None:
self.n_targets = len(targets)
else:
self.n_targets = 2
self.confusion_matrix = ConfusionMatrix(self.n_targets, dtype)
self.last_class = None
self.rranks = FastBuffer(window_size)
self.targets = targets
self.window_size = window_size
self.true_labels = FastBuffer(window_size)
self.predictions = FastBuffer(window_size)
self.temp = 0
self.last_prediction = None
self.last_true_label = None
self.last_sample = None
self.majority_classifier = 0
self.correct_no_change = 0
self.majority_classifier_correction = FastBuffer(window_size)
self.correct_no_change_correction = FastBuffer(window_size)
def reset(self):
if self.targets is not None:
self.n_targets = len(self.targets)
else:
self.n_targets = 2
self.true_labels = FastBuffer(self.window_size)
self.predictions = FastBuffer(self.window_size)
self.temp = 0
self.last_prediction = None
self.last_true_label = None
self.last_sample = None
self.majority_classifier = 0
self.correct_no_change = 0
self.confusion_matrix.restart(self.n_targets)
self.majority_classifier_correction = FastBuffer(self.window_size)
self.correct_no_change_correction = FastBuffer(self.window_size)
self.rranks = FastBuffer(self.window_size)
def add_result(self, y_true, y_pred, weight=1.0):
""" Updates its statistics with the results of a prediction.
If needed it will remove samples from the observation window.
Parameters
----------
y_true: int
Index of the true label
y_pred: int
Indices of top-N predicted labels
weight: float
Sample's weight
"""
check_weights(weight)
old_true = self.true_labels.add_element(np.array([y_true]))
rank = np.where(y_pred == y_true)[0]
if rank.size == 1: # Relevant item exists
self.rranks.add_element(np.array([1 / (rank[0] + 1)]))
self.confusion_matrix.update(y_true, y_true, weight=weight)
old_predict = self.predictions.add_element(np.array([y_true])) # Add correct prediction
else:
self.rranks.add_element(np.array([0]))
self.confusion_matrix.update(y_true, y_pred[0], weight=weight)
old_predict = self.predictions.add_element(np.array([y_pred[0]])) # Add some incorrect prediction
# Verify if it's needed to decrease the count of any label
# pair in the confusion matrix
if (old_true is not None) and (old_predict is not None):
self.temp += 1
self.confusion_matrix.remove(old_true[0], old_predict[0])
self.correct_no_change += self.correct_no_change_correction.peek()
# Verify if it's needed to decrease the correct_no_change
if (self.last_true_label == y_true) and (self.last_true_label is not None):
self.correct_no_change += weight
self.correct_no_change_correction.add_element([-1])
else:
self.correct_no_change_correction.add_element([0])
self.last_true_label = y_true
self.last_prediction = y_pred
def get_last(self):
return self.last_true_label, self.last_prediction
def get_accuracy(self):
""" Computes the window/current accuracy.
Returns
-------
float
The window/current accuracy.
"""
sum_value = self.confusion_matrix.get_sum_main_diagonal()
try:
if sum_value > self.window_size:
print('sum value: {}'.format(sum_value))
print('window size: {}'.format(self.window_size))
return sum_value / self.true_labels.get_current_size()
except ZeroDivisionError:
return 0.0
def get_f1_score(self):
""" Compute the F1-score of the classifier.
Returns
-------
float
The F1-score
"""
precision = self.get_precision()
recall = self.get_recall()
if recall + precision == 0:
return 0.0
else:
return 2 * (precision * recall) / (precision + recall)
def get_precision(self):
""" compute the precision of the classifier.
Returns
-------
float
The precision
"""
return self.get_accuracy() / Data.rec_size
def get_recall(self):
""" Compute the recall of the classifier..
Returns
-------
float
The recall.
"""
return self.get_accuracy()
def get_mrr(self):
""" Compute the mean reciprocal rank of the classifier.
Returns
-------
float
The mean reciprocal rank.
"""
return sum(self.rranks.get_queue()) / self.rranks.get_current_size()
@property
def _matrix(self):
return self.confusion_matrix.matrix
@property
def sample_count(self):
return self.true_labels.get_current_size()
def get_info(self):
return '{}:'.format(type(self).__name__) + \
' - sample_count: {}'.format(self.sample_count) + \
' - window_size: {}'.format(self.window_size) + \
' - f1-score: {:.6f}'.format(self.get_f1_score()) + \
' - precision: {:.6f}'.format(self.get_precision()) + \
' - recall: {:.6f}'.format(self.get_recall())
class WindowMultiTargetRegressionMeasurements(object):
""" This class is used to keep updated statistics over a multi-target regression
learner in a multi-target regression problem context inside a fixed sized
window. It uses FastBuffer objects to simulate the fixed sized windows.
It will keep track of partial metrics, that can be provided at
any moment. The relevant metrics kept by an instance of this class
are: AMSE (average mean square error), AMAE (average mean absolute error),
and ARMSE (average root mean square error).
"""
def __init__(self, window_size=200):
super().__init__()
self.n_targets = 0
self.total_square_error = 0.0
self.average_error = 0.0
self.last_true_label = None
self.last_prediction = None
self.total_square_error_correction = FastBuffer(window_size)
self.average_error_correction = FastBuffer(window_size)
self.window_size = window_size
def reset(self):
self.total_square_error = 0.0
self.average_error = 0.0
self.last_true_label = None
self.last_prediction = None
self.total_square_error_correction = FastBuffer(self.window_size)
self.average_error_correction = FastBuffer(self.window_size)
def add_result(self, y, prediction):
""" Use the true value and the prediction to update the statistics.
Parameters
----------
y: float or list or np.ndarray
The true value(s).
prediction: float or list or np.ndarray
The predicted value(s).
prediction: float or list or np.ndarray
The predicted value(s).
"""
self.last_true_label = y
self.last_prediction = prediction
m = 0
if hasattr(y, 'size'):
m = y.size
elif hasattr(y, 'append'):
m = len(y)
self.n_targets = m
self.total_square_error += (y - prediction) ** 2
self.average_error += np.absolute(y - prediction)
old_square = self.total_square_error_correction.add_element(
np.array([-1 * ((y - prediction) ** 2)])
)
old_average = self.average_error_correction.add_element(
np.array([-1 * (np.absolute(y - prediction))])
)
if (old_square is not None) and (old_average is not None):
self.total_square_error += old_square[0]
self.average_error += old_average[0]
def get_average_mean_square_error(self):
""" Computes the window/current average mean square error.
Returns
-------
float
The window/current average mean square error.
"""
if self._sample_count == 0:
return 0.0
else:
return np.sum(self.total_square_error / self._sample_count) \
/ self.n_targets
def get_average_absolute_error(self):
""" Computes the window/current average mean absolute error.
Returns
-------
float
The window/current average mean absolute error.
"""
if self._sample_count == 0:
return 0.0
else:
return np.sum(self.average_error / self._sample_count) \
/ self.n_targets
def get_average_root_mean_square_error(self):
""" Computes the mean square error.
Returns
-------
float
The average mean square error.
"""
if self._sample_count == 0:
return 0.0
else:
return np.sum(np.sqrt(self.total_square_error /
self._sample_count)) \
/ self.n_targets
def get_last(self):
return self.last_true_label, self.last_prediction
@property
def _sample_count(self):
return self.total_square_error_correction.get_current_size()
def get_info(self):
return 'MultiTargetRegressionMeasurements: sample_count: ' + \
str(self._sample_count) + ' - average_mean_square_error: ' + \
str(self.get_average_mean_square_error()) + ' - average_mean_absolute_error: ' + \
str(self.get_average_absolute_error()) + ' - average_root_mean_square_error: ' + \
str(self.get_average_root_mean_square_error())