class Precision(TrainExtension): def __init__(self): self.predictor = None self.precision_list = [] def setup(self, model, dataset, algorithm): self.predictor = Predictor(model) def on_monitor(self, model, dataset, algorithm, stat_dic=None): import sys sys.path.append('..') from utils import values if stat_dic is None: from utils.casting import label_lists2types # obtaining validating set valid_x = algorithm.monitoring_dataset['valid'].X valid_y = algorithm.monitoring_dataset['valid'].y y_pred = self.predictor.get_predictions(valid_x) stat_dic = label_lists2types(valid_y, y_pred) # precision = TP/(TP + FP) if stat_dic[values.TP] == 0: precision = 0 else: precision = float(stat_dic[values.TP])/(stat_dic[values.TP] + stat_dic[values.FP]) self.precision_list.append(precision) print "precision:", precision
class Accuracy(TrainExtension): def __init__(self): self.predictor = None self.accuracy_list = [] def setup(self, model, dataset, algorithm): self.predictor = Predictor(model) def on_monitor(self, model, dataset, algorithm, stat_dic=None): import sys sys.path.append('..') from utils import values if stat_dic is None: from utils.casting import label_lists2types # obtaining validating set valid_x = algorithm.monitoring_dataset['valid'].X valid_y = algorithm.monitoring_dataset['valid'].y y_pred = self.predictor.get_predictions(valid_x) stat_dic = label_lists2types(valid_y, y_pred) # accuracy = (TP + TN) / TOTAL if (stat_dic[values.TP] + stat_dic[values.TN]) == 0: accuracy = 0 else: accuracy = float(stat_dic[values.TP]+stat_dic[values.TN])/sum(stat_dic.values()) self.accuracy_list.append(accuracy) print "accuracy:", accuracy
class StatisticsSymmetricThreshold(TrainExtension): def __init__(self, call_list): """ :type call_list: list containing objects to calculate certain statistics """ self.predictor = None self.symmetric_threshold = None self.call_list = call_list def setup(self, model, dataset, algorithm): self.predictor = Predictor(model) self.symmetric_threshold = SymmetricThresholdWRTF1Score() for extension in self.call_list: extension.setup(model, dataset, algorithm) def on_monitor(self, model, dataset, algorithm): import sys sys.path.append('..') from utils.casting import label_lists2types from utils import values print '\nSHOWING STATISTICS FOR SYMMETRIC THRESHOLD' valid_x = algorithm.monitoring_dataset['valid'].X valid_y = algorithm.monitoring_dataset['valid'].y y_pred = self.predictor.get_predictions(valid_x) threshold, score = self.symmetric_threshold.compute_optimal_threshold_and_score(valid_y, y_pred) print "Best threshold", threshold, '\ncorresponding F1Score:', score stat_dic = label_lists2types(valid_y, y_pred, sym_t=threshold) print values.TP, ':', stat_dic[values.TP], '\t\t', values.TN, ':', stat_dic[values.TN], '\n', \ values.FP, ':', stat_dic[values.FP], '\t\t', values.FN, ':', stat_dic[values.FN], '\n', \ values.FNP, ':', stat_dic[values.FNP], '\t\t', values.FNN, ':', stat_dic[values.FNN] for extension in self.call_list: # TODO: consider using inspect.getargspec() try: extension.on_monitor(model, dataset, algorithm, stat_dic) except TypeError: extension.on_monitor(model, dataset, algorithm)
class F1Score(TrainExtension): def __init__(self, save_best_model_path=None, save=False): self.predictor = None self.score_list = [] self.debug = True self.saving_path = save_best_model_path self.save = save def setup(self, model, dataset, algorithm): self.predictor = Predictor(model) def on_monitor(self, model, dataset, algorithm): import numpy as np # this shall never happen but better safe than sorry if self.predictor is None: self.setup(model, dataset, algorithm) # obtaining validating set # valid_x = algorithm.monitoring_dataset['valid'].X valid_y = algorithm.monitoring_dataset['valid'].y y_pred = self.predictor.get_predictions(valid_x) y_classes = [np.argmax(pred) for pred in y_pred] score = f1_score(y_true=valid_y, y_pred=y_classes) self.score_list.append(score) if self.saving_path is not None and self.save: if max(self.score_list) == score: try: # Make sure that saving does not serialize the dataset dataset._serialization_guard = SerializationGuard() save_path = self.saving_path serial.save(save_path, model, on_overwrite='backup') finally: dataset._serialization_guard = None print "F1 score:", score
class StatisticsNoThreshold(TrainExtension): def __init__(self, call_list): """ :type call_list: list containing objects to calculate certain statistics """ self.predictor = None self.call_list = call_list def setup(self, model, dataset, algorithm): self.predictor = Predictor(model) for extension in self.call_list: extension.setup(model, dataset, algorithm) def on_monitor(self, model, dataset, algorithm): import sys sys.path.append('..') from utils.casting import label_lists2types from utils import values print '\nSHOWING STATISTICS FOR NO THRESHOLD' valid_x = algorithm.monitoring_dataset['valid'].X valid_y = algorithm.monitoring_dataset['valid'].y y_pred = self.predictor.get_predictions(valid_x) stat_dic = label_lists2types(valid_y, y_pred) print values.TP, ':', stat_dic[values.TP], '\t\t', values.TN, ':', stat_dic[values.TN], '\n', \ values.FP, ':', stat_dic[values.FP], '\t\t', values.FN, ':', stat_dic[values.FN], '\n', \ values.FNP, ':', stat_dic[values.FNP], '\t\t', values.FNN, ':', stat_dic[values.FNN] for extension in self.call_list: # TODO: consider using inspect.getargspec() try: extension.on_monitor(model, dataset, algorithm, stat_dic) except TypeError: extension.on_monitor(model, dataset, algorithm)
class SymmetricThresholdWRTF1Score(F1Score): def __init__(self, save_best_model_path=None, save=False): super(SymmetricThresholdWRTF1Score, self).__init__() self.threshold_list = [] self.saving_path = save_best_model_path self.save = save def setup(self, model, dataset, algorithm): self.predictor = Predictor(model) def on_monitor(self, model, dataset, algorithm): # this shall never happen but better safe than sorry if self.predictor is None: self.setup(model, dataset, algorithm) # obtaining validating set # TODO: finally we want to have train-validation-test set. Or sth. valid_x = algorithm.monitoring_dataset['valid'].X valid_y = algorithm.monitoring_dataset['valid'].y predictions = self.predictor.get_predictions(valid_x) threshold, score = self.compute_optimal_threshold_and_score(valid_y, predictions) self.threshold_list.append(threshold) self.score_list.append(score) if self.saving_path is not None and self.save: if max(self.score_list) == score: try: # Make sure that saving does not serialize the dataset dataset._serialization_guard = SerializationGuard() save_path = self.saving_path serial.save(save_path, model, on_overwrite='backup') finally: dataset._serialization_guard = None print "F1Score1Threshold score", score, "\ncorresponding threshold:", threshold @staticmethod def compute_optimal_threshold_and_score(true_y, predictions): # F1_score = 2TP/(2TP + FN + FP) # our F1_score to deal with not classifying some examples: # if some example isn't classified (is between thresholds) it counts as 1/2 of classifying it wrongly # therefore the F1_score becomes: 2TP/(2TP + FN +FP + 0.5 * unclassified) from numpy import argmax, mean import sys sys.path.append('..') from utils import values dic = {} # the threshold will be symmetric, so we only care about how far away is the prediction from 0.5 # therefore we fold the dictionary in half for index in xrange(len(true_y)): if argmax(predictions[index]) != true_y[index]: # FALSE NEGATIVE OR FALSE POSITIVE # floating to get a hashable float instead of unhashable numpy array # we also need max max because numpy... dic[float(abs(0.5 - max(max(predictions[index]))))] = values.FN_FP elif argmax(predictions[index]) == 1: # TRUE POSITIVE dic[float(abs(0.5 - max(max(predictions[index]))))] = values.TP # else TRUE NEGATIVE, we have no interest in this one TP = sum(1 for x in dic.values() if x == values.TP) FP_FN = len(dic)-TP unclassified = 0 scores = [] sorted_dic_keys = sorted(dic) # don't want to sort dic over and over again for key in sorted_dic_keys: unclassified += 1 if dic[key] == values.FN_FP: # it was FN or FP FP_FN -= 1 else: TP -= 1 # it was TP scores.append(2*TP/(2*TP + FP_FN + 0.5 * unclassified)) best_score_index = argmax(scores) max_score = scores[best_score_index] t1 = sorted_dic_keys[best_score_index] t2 = sorted_dic_keys[best_score_index-1] best_threshold = 0.5 + mean([t1, t2]) # adding 0.5 as the dictionary has been folded in half return best_threshold, max_score
def setup(self, model, dataset, algorithm): self.predictor = Predictor(model)
def setup(self, model, dataset, algorithm): self.predictor = Predictor(model) self.symmetric_threshold = SymmetricThresholdWRTF1Score() for extension in self.call_list: extension.setup(model, dataset, algorithm)
def setup(self, model, dataset, algorithm): self.predictor = Predictor(model) for extension in self.call_list: extension.setup(model, dataset, algorithm)
class ROC_Yoduen(F1Score): def __init__(self, save_best_model_path=None, save=False): super(ROC_Yoduen, self).__init__() self.threshold_list = [] self.saving_path = save_best_model_path self.save = save def setup(self, model, dataset, algorithm): self.predictor = Predictor(model) def on_monitor(self, model, dataset, algorithm): # this shall never happen but better safe than sorry if self.predictor is None: self.setup(model, dataset, algorithm) # obtaining validating set # TODO: finally we want to have train-validation-test set. Or sth. valid_x = algorithm.monitoring_dataset['valid'].X valid_y = algorithm.monitoring_dataset['valid'].y predictions = self.predictor.get_predictions(valid_x) best_threshold, best_score = self.compute_optimal_threshold_and_score(valid_y, predictions) self.threshold_list.append(best_threshold) self.score_list.append(best_score) if self.saving_path is not None and self.save: if max(self.score_list) == best_score: try: # Make sure that saving does not serialize the dataset dataset._serialization_guard = SerializationGuard() # TODO: this should actually be: save_path = self.saving_path save_path = 'best_model_roc_youden.model' serial.save(save_path, model, on_overwrite='backup') finally: dataset._serialization_guard = None stat_dic = types_dict(valid_y, predictions, threshold=best_threshold) print '\nSHOWING STATISTICS FOR ROC with Youden metric' print "ROC using Youden metric\nscore:", best_score, "\ncorresponding threshold:", best_threshold print values.TP, ':', stat_dic[values.TP], '\t\t', values.TN, ':', stat_dic[values.TN], '\n', \ values.FP, ':', stat_dic[values.FP], '\t\t', values.FN, ':', stat_dic[values.FN], '\n', \ values.FNP, ':', stat_dic[values.FNP], '\t\t', values.FNN, ':', stat_dic[values.FNN] precision = float(stat_dic[values.TP])/(stat_dic[values.TP] + stat_dic[values.FP]) recall = float(stat_dic[values.TP])/(stat_dic[values.TP] + stat_dic[values.FN]) f1score = 0 if precision+recall != 0: f1score = 2*precision*recall/(precision+recall) print 'precision:', precision print "recall:", recall print "f1score", f1score @staticmethod def compute_optimal_threshold_and_score(true_y, predictions): axis = sorted([[pred[0][1], label] for pred, label in zip(predictions, true_y)]) # active 1 [[ 0. 1. 0. ]] [[ 0. 1. ]] # nonactive 0 [[ 1. 0. 0. ]] [[ 1. 0. ]] # middle -1 [[ 0. 0. 1. ]] actives = sum([1 for arr_pred_lab in axis if arr_pred_lab[1] == 1]) nonactives = len(axis) - actives # threshold is zero, it means we label ALL the samples as negatives TP = actives FP = nonactives TN = 0 FN = 0 best_score = 0 best_threshold = float(0) next_pred_after_best_threshold = 0 update_next = False for prediction, label in axis: if label[0] == 1: TP -= 1 # after moving threshold we have one well classified positive example less FN += 1 # and that means we classify wrongly one positive example more else: # label is 0 - means nonactive TN += 1 # we have one well classified negative example more FP -= 1 # so that's one wrongly classified negative example less # calculating score according to Youden's metric score = (float(TP)/(float(TP) + float(FN))) - (float(FP)/(float(FP) + float(TN))) # print 'TP:', TP, '\tFP:', FP, '\nFN:', FN, '\tTN:', TN # print 'score:', score, 'threshold:', prediction, '\n\n' if update_next: update_next = False next_pred_after_best_threshold = prediction if score == best_score: pass #print 'TP:', TP, '\tFP:', FP, '\nFN:', FN, '\tTN:', TN if score > best_score: #print 'TP:', TP, '\tFP:', FP, '\nFN:', FN, '\tTN:', TN best_score = score best_threshold = prediction update_next = True # compute optimal threshold best_threshold = (best_threshold + next_pred_after_best_threshold)/2.0 return best_threshold, best_score
class TwoThresholdWRTF1Score(F1Score): def __init__(self, save_best_model_path=None, save=False): super(TwoThresholdWRTF1Score, self).__init__() self.thresholds_list = [] self.saving_path = save_best_model_path self.save = save def setup(self, model, dataset, algorithm): self.predictor = Predictor(model) def on_monitor(self, model, dataset, algorithm): # this shall never happen but better safe than sorry if self.predictor is None: self.setup(model, dataset, algorithm) # obtaining validating set # TODO: finally we want to have train-validation-test set. Or sth. valid_x = algorithm.monitoring_dataset['valid'].X valid_y = algorithm.monitoring_dataset['valid'].y predictions = self.predictor.get_predictions(valid_x) lower_threshold, upper_threshold, score = self.compute_optimal_threshold_and_score(valid_y, predictions) self.thresholds_list.append((upper_threshold, lower_threshold)) self.score_list.append(score) if self.saving_path is not None and self.save: if max(self.score_list) == score: try: # Make sure that saving does not serialize the dataset dataset._serialization_guard = SerializationGuard() save_path = self.saving_path serial.save(save_path, model, on_overwrite='backup') finally: dataset._serialization_guard = None print "\n\nTwoThreshold score", score, "\ncorresponding threshold pair:", lower_threshold, ':', upper_threshold @staticmethod def compute_optimal_threshold_and_score(true_y, predictions): axis = sorted([[pred[0][1], label] for pred, label in zip(predictions, true_y)]) x = len(axis) y = 0 while y < x and axis[y][1] == 0: y += 1 while x > y and axis[x-1][1] == 1: x -= 1 tn_global = y tp_global = len(axis) - x fn_global = 0 fp_global = 0 maximal_score = -1.0 best_y_idx = y best_x_idx = x for i in xrange(y, x+1): tp_local = tp_global fp_local = fp_global for j in xrange(x, i-1, -1): middle = len(axis) - tp_local - fp_local - tn_global - fn_global if tp_local == 0: score_local = 0 else: score_local = 2.0 * tp_local / (2 * tp_local + fp_local + fn_global + 0.5 * middle) if score_local > maximal_score: maximal_score = score_local best_y_idx, best_x_idx = i, j if axis[j-1][1] == 1: tp_local += 1 else: fp_local += 1 if i == x: break if axis[i][1] == 0: tn_global += 1 else: fn_global += 1 best_y = 0 if best_y_idx == 0 else 1 if len(axis) > best_y_idx > 0: best_y = (axis[best_y_idx-1][0] + axis[best_y_idx][0]) / 2.0 best_x = 0 if best_x_idx == 0 else 1 if len(axis) > best_x_idx > 0: best_x = (axis[best_x_idx-1][0] + axis[best_x_idx][0]) / 2.0 return best_y, best_x, maximal_score