class Precision(TrainExtension):
def __init__(self):
self.predictor = None
self.precision_list = []
def setup(self, model, dataset, algorithm):
self.predictor = Predictor(model)
def on_monitor(self, model, dataset, algorithm, stat_dic=None):
import sys
sys.path.append('..')
from utils import values
if stat_dic is None:
from utils.casting import label_lists2types
# obtaining validating set
valid_x = algorithm.monitoring_dataset['valid'].X
valid_y = algorithm.monitoring_dataset['valid'].y
y_pred = self.predictor.get_predictions(valid_x)
stat_dic = label_lists2types(valid_y, y_pred)
# precision = TP/(TP + FP)
if stat_dic[values.TP] == 0:
precision = 0
else:
precision = float(stat_dic[values.TP])/(stat_dic[values.TP] + stat_dic[values.FP])
self.precision_list.append(precision)
print "precision:", precision
class Accuracy(TrainExtension):
def __init__(self):
self.predictor = None
self.accuracy_list = []
def setup(self, model, dataset, algorithm):
self.predictor = Predictor(model)
def on_monitor(self, model, dataset, algorithm, stat_dic=None):
import sys
sys.path.append('..')
from utils import values
if stat_dic is None:
from utils.casting import label_lists2types
# obtaining validating set
valid_x = algorithm.monitoring_dataset['valid'].X
valid_y = algorithm.monitoring_dataset['valid'].y
y_pred = self.predictor.get_predictions(valid_x)
stat_dic = label_lists2types(valid_y, y_pred)
# accuracy = (TP + TN) / TOTAL
if (stat_dic[values.TP] + stat_dic[values.TN]) == 0:
accuracy = 0
else:
accuracy = float(stat_dic[values.TP]+stat_dic[values.TN])/sum(stat_dic.values())
self.accuracy_list.append(accuracy)
print "accuracy:", accuracy
class StatisticsSymmetricThreshold(TrainExtension):
def __init__(self, call_list):
"""
:type call_list: list containing objects to calculate certain statistics
"""
self.predictor = None
self.symmetric_threshold = None
self.call_list = call_list
def setup(self, model, dataset, algorithm):
self.predictor = Predictor(model)
self.symmetric_threshold = SymmetricThresholdWRTF1Score()
for extension in self.call_list:
extension.setup(model, dataset, algorithm)
def on_monitor(self, model, dataset, algorithm):
import sys
sys.path.append('..')
from utils.casting import label_lists2types
from utils import values
print '\nSHOWING STATISTICS FOR SYMMETRIC THRESHOLD'
valid_x = algorithm.monitoring_dataset['valid'].X
valid_y = algorithm.monitoring_dataset['valid'].y
y_pred = self.predictor.get_predictions(valid_x)
threshold, score = self.symmetric_threshold.compute_optimal_threshold_and_score(valid_y, y_pred)
print "Best threshold", threshold, '\ncorresponding F1Score:', score
stat_dic = label_lists2types(valid_y, y_pred, sym_t=threshold)
print values.TP, ':', stat_dic[values.TP], '\t\t', values.TN, ':', stat_dic[values.TN], '\n', \
values.FP, ':', stat_dic[values.FP], '\t\t', values.FN, ':', stat_dic[values.FN], '\n', \
values.FNP, ':', stat_dic[values.FNP], '\t\t', values.FNN, ':', stat_dic[values.FNN]
for extension in self.call_list:
# TODO: consider using inspect.getargspec()
try:
extension.on_monitor(model, dataset, algorithm, stat_dic)
except TypeError:
extension.on_monitor(model, dataset, algorithm)
class F1Score(TrainExtension):
def __init__(self, save_best_model_path=None, save=False):
self.predictor = None
self.score_list = []
self.debug = True
self.saving_path = save_best_model_path
self.save = save
def setup(self, model, dataset, algorithm):
self.predictor = Predictor(model)
def on_monitor(self, model, dataset, algorithm):
import numpy as np
# this shall never happen but better safe than sorry
if self.predictor is None:
self.setup(model, dataset, algorithm)
# obtaining validating set #
valid_x = algorithm.monitoring_dataset['valid'].X
valid_y = algorithm.monitoring_dataset['valid'].y
y_pred = self.predictor.get_predictions(valid_x)
y_classes = [np.argmax(pred) for pred in y_pred]
score = f1_score(y_true=valid_y, y_pred=y_classes)
self.score_list.append(score)
if self.saving_path is not None and self.save:
if max(self.score_list) == score:
try:
# Make sure that saving does not serialize the dataset
dataset._serialization_guard = SerializationGuard()
save_path = self.saving_path
serial.save(save_path, model,
on_overwrite='backup')
finally:
dataset._serialization_guard = None
print "F1 score:", score
class StatisticsNoThreshold(TrainExtension):
def __init__(self, call_list):
"""
:type call_list: list containing objects to calculate certain statistics
"""
self.predictor = None
self.call_list = call_list
def setup(self, model, dataset, algorithm):
self.predictor = Predictor(model)
for extension in self.call_list:
extension.setup(model, dataset, algorithm)
def on_monitor(self, model, dataset, algorithm):
import sys
sys.path.append('..')
from utils.casting import label_lists2types
from utils import values
print '\nSHOWING STATISTICS FOR NO THRESHOLD'
valid_x = algorithm.monitoring_dataset['valid'].X
valid_y = algorithm.monitoring_dataset['valid'].y
y_pred = self.predictor.get_predictions(valid_x)
stat_dic = label_lists2types(valid_y, y_pred)
print values.TP, ':', stat_dic[values.TP], '\t\t', values.TN, ':', stat_dic[values.TN], '\n', \
values.FP, ':', stat_dic[values.FP], '\t\t', values.FN, ':', stat_dic[values.FN], '\n', \
values.FNP, ':', stat_dic[values.FNP], '\t\t', values.FNN, ':', stat_dic[values.FNN]
for extension in self.call_list:
# TODO: consider using inspect.getargspec()
try:
extension.on_monitor(model, dataset, algorithm, stat_dic)
except TypeError:
extension.on_monitor(model, dataset, algorithm)
class SymmetricThresholdWRTF1Score(F1Score):
def __init__(self, save_best_model_path=None, save=False):
super(SymmetricThresholdWRTF1Score, self).__init__()
self.threshold_list = []
self.saving_path = save_best_model_path
self.save = save
def setup(self, model, dataset, algorithm):
self.predictor = Predictor(model)
def on_monitor(self, model, dataset, algorithm):
# this shall never happen but better safe than sorry
if self.predictor is None:
self.setup(model, dataset, algorithm)
# obtaining validating set # TODO: finally we want to have train-validation-test set. Or sth.
valid_x = algorithm.monitoring_dataset['valid'].X
valid_y = algorithm.monitoring_dataset['valid'].y
predictions = self.predictor.get_predictions(valid_x)
threshold, score = self.compute_optimal_threshold_and_score(valid_y, predictions)
self.threshold_list.append(threshold)
self.score_list.append(score)
if self.saving_path is not None and self.save:
if max(self.score_list) == score:
try:
# Make sure that saving does not serialize the dataset
dataset._serialization_guard = SerializationGuard()
save_path = self.saving_path
serial.save(save_path, model,
on_overwrite='backup')
finally:
dataset._serialization_guard = None
print "F1Score1Threshold score", score, "\ncorresponding threshold:", threshold
@staticmethod
def compute_optimal_threshold_and_score(true_y, predictions):
# F1_score = 2TP/(2TP + FN + FP)
# our F1_score to deal with not classifying some examples:
# if some example isn't classified (is between thresholds) it counts as 1/2 of classifying it wrongly
# therefore the F1_score becomes: 2TP/(2TP + FN +FP + 0.5 * unclassified)
from numpy import argmax, mean
import sys
sys.path.append('..')
from utils import values
dic = {}
# the threshold will be symmetric, so we only care about how far away is the prediction from 0.5
# therefore we fold the dictionary in half
for index in xrange(len(true_y)):
if argmax(predictions[index]) != true_y[index]: # FALSE NEGATIVE OR FALSE POSITIVE
# floating to get a hashable float instead of unhashable numpy array
# we also need max max because numpy...
dic[float(abs(0.5 - max(max(predictions[index]))))] = values.FN_FP
elif argmax(predictions[index]) == 1: # TRUE POSITIVE
dic[float(abs(0.5 - max(max(predictions[index]))))] = values.TP
# else TRUE NEGATIVE, we have no interest in this one
TP = sum(1 for x in dic.values() if x == values.TP)
FP_FN = len(dic)-TP
unclassified = 0
scores = []
sorted_dic_keys = sorted(dic) # don't want to sort dic over and over again
for key in sorted_dic_keys:
unclassified += 1
if dic[key] == values.FN_FP: # it was FN or FP
FP_FN -= 1
else:
TP -= 1 # it was TP
scores.append(2*TP/(2*TP + FP_FN + 0.5 * unclassified))
best_score_index = argmax(scores)
max_score = scores[best_score_index]
t1 = sorted_dic_keys[best_score_index]
t2 = sorted_dic_keys[best_score_index-1]
best_threshold = 0.5 + mean([t1, t2]) # adding 0.5 as the dictionary has been folded in half
return best_threshold, max_score
def setup(self, model, dataset, algorithm):
self.predictor = Predictor(model)
def setup(self, model, dataset, algorithm):
self.predictor = Predictor(model)
self.symmetric_threshold = SymmetricThresholdWRTF1Score()
for extension in self.call_list:
extension.setup(model, dataset, algorithm)
def setup(self, model, dataset, algorithm):
self.predictor = Predictor(model)
for extension in self.call_list:
extension.setup(model, dataset, algorithm)
class ROC_Yoduen(F1Score):
def __init__(self, save_best_model_path=None, save=False):
super(ROC_Yoduen, self).__init__()
self.threshold_list = []
self.saving_path = save_best_model_path
self.save = save
def setup(self, model, dataset, algorithm):
self.predictor = Predictor(model)
def on_monitor(self, model, dataset, algorithm):
# this shall never happen but better safe than sorry
if self.predictor is None:
self.setup(model, dataset, algorithm)
# obtaining validating set # TODO: finally we want to have train-validation-test set. Or sth.
valid_x = algorithm.monitoring_dataset['valid'].X
valid_y = algorithm.monitoring_dataset['valid'].y
predictions = self.predictor.get_predictions(valid_x)
best_threshold, best_score = self.compute_optimal_threshold_and_score(valid_y, predictions)
self.threshold_list.append(best_threshold)
self.score_list.append(best_score)
if self.saving_path is not None and self.save:
if max(self.score_list) == best_score:
try:
# Make sure that saving does not serialize the dataset
dataset._serialization_guard = SerializationGuard()
# TODO: this should actually be: save_path = self.saving_path
save_path = 'best_model_roc_youden.model'
serial.save(save_path, model,
on_overwrite='backup')
finally:
dataset._serialization_guard = None
stat_dic = types_dict(valid_y, predictions, threshold=best_threshold)
print '\nSHOWING STATISTICS FOR ROC with Youden metric'
print "ROC using Youden metric\nscore:", best_score, "\ncorresponding threshold:", best_threshold
print values.TP, ':', stat_dic[values.TP], '\t\t', values.TN, ':', stat_dic[values.TN], '\n', \
values.FP, ':', stat_dic[values.FP], '\t\t', values.FN, ':', stat_dic[values.FN], '\n', \
values.FNP, ':', stat_dic[values.FNP], '\t\t', values.FNN, ':', stat_dic[values.FNN]
precision = float(stat_dic[values.TP])/(stat_dic[values.TP] + stat_dic[values.FP])
recall = float(stat_dic[values.TP])/(stat_dic[values.TP] + stat_dic[values.FN])
f1score = 0
if precision+recall != 0:
f1score = 2*precision*recall/(precision+recall)
print 'precision:', precision
print "recall:", recall
print "f1score", f1score
@staticmethod
def compute_optimal_threshold_and_score(true_y, predictions):
axis = sorted([[pred[0][1], label] for pred, label in zip(predictions, true_y)])
# active 1 [[ 0. 1. 0. ]] [[ 0. 1. ]]
# nonactive 0 [[ 1. 0. 0. ]] [[ 1. 0. ]]
# middle -1 [[ 0. 0. 1. ]]
actives = sum([1 for arr_pred_lab in axis if arr_pred_lab[1] == 1])
nonactives = len(axis) - actives
# threshold is zero, it means we label ALL the samples as negatives
TP = actives
FP = nonactives
TN = 0
FN = 0
best_score = 0
best_threshold = float(0)
next_pred_after_best_threshold = 0
update_next = False
for prediction, label in axis:
if label[0] == 1:
TP -= 1 # after moving threshold we have one well classified positive example less
FN += 1 # and that means we classify wrongly one positive example more
else: # label is 0 - means nonactive
TN += 1 # we have one well classified negative example more
FP -= 1 # so that's one wrongly classified negative example less
# calculating score according to Youden's metric
score = (float(TP)/(float(TP) + float(FN))) - (float(FP)/(float(FP) + float(TN)))
# print 'TP:', TP, '\tFP:', FP, '\nFN:', FN, '\tTN:', TN
# print 'score:', score, 'threshold:', prediction, '\n\n'
if update_next:
update_next = False
next_pred_after_best_threshold = prediction
if score == best_score:
pass
#print 'TP:', TP, '\tFP:', FP, '\nFN:', FN, '\tTN:', TN
if score > best_score:
#print 'TP:', TP, '\tFP:', FP, '\nFN:', FN, '\tTN:', TN
best_score = score
best_threshold = prediction
update_next = True
# compute optimal threshold
best_threshold = (best_threshold + next_pred_after_best_threshold)/2.0
return best_threshold, best_score
class TwoThresholdWRTF1Score(F1Score):
def __init__(self, save_best_model_path=None, save=False):
super(TwoThresholdWRTF1Score, self).__init__()
self.thresholds_list = []
self.saving_path = save_best_model_path
self.save = save
def setup(self, model, dataset, algorithm):
self.predictor = Predictor(model)
def on_monitor(self, model, dataset, algorithm):
# this shall never happen but better safe than sorry
if self.predictor is None:
self.setup(model, dataset, algorithm)
# obtaining validating set # TODO: finally we want to have train-validation-test set. Or sth.
valid_x = algorithm.monitoring_dataset['valid'].X
valid_y = algorithm.monitoring_dataset['valid'].y
predictions = self.predictor.get_predictions(valid_x)
lower_threshold, upper_threshold, score = self.compute_optimal_threshold_and_score(valid_y, predictions)
self.thresholds_list.append((upper_threshold, lower_threshold))
self.score_list.append(score)
if self.saving_path is not None and self.save:
if max(self.score_list) == score:
try:
# Make sure that saving does not serialize the dataset
dataset._serialization_guard = SerializationGuard()
save_path = self.saving_path
serial.save(save_path, model,
on_overwrite='backup')
finally:
dataset._serialization_guard = None
print "\n\nTwoThreshold score", score, "\ncorresponding threshold pair:", lower_threshold, ':', upper_threshold
@staticmethod
def compute_optimal_threshold_and_score(true_y, predictions):
axis = sorted([[pred[0][1], label] for pred, label in zip(predictions, true_y)])
x = len(axis)
y = 0
while y < x and axis[y][1] == 0:
y += 1
while x > y and axis[x-1][1] == 1:
x -= 1
tn_global = y
tp_global = len(axis) - x
fn_global = 0
fp_global = 0
maximal_score = -1.0
best_y_idx = y
best_x_idx = x
for i in xrange(y, x+1):
tp_local = tp_global
fp_local = fp_global
for j in xrange(x, i-1, -1):
middle = len(axis) - tp_local - fp_local - tn_global - fn_global
if tp_local == 0:
score_local = 0
else:
score_local = 2.0 * tp_local / (2 * tp_local + fp_local + fn_global + 0.5 * middle)
if score_local > maximal_score:
maximal_score = score_local
best_y_idx, best_x_idx = i, j
if axis[j-1][1] == 1:
tp_local += 1
else:
fp_local += 1
if i == x:
break
if axis[i][1] == 0:
tn_global += 1
else:
fn_global += 1
best_y = 0 if best_y_idx == 0 else 1
if len(axis) > best_y_idx > 0:
best_y = (axis[best_y_idx-1][0] + axis[best_y_idx][0]) / 2.0
best_x = 0 if best_x_idx == 0 else 1
if len(axis) > best_x_idx > 0:
best_x = (axis[best_x_idx-1][0] + axis[best_x_idx][0]) / 2.0
return best_y, best_x, maximal_score
