def on_monitor(self, model, dataset, algorithm): import sys sys.path.append('..') from utils.casting import label_lists2types from utils import values print '\nSHOWING STATISTICS FOR SYMMETRIC THRESHOLD' valid_x = algorithm.monitoring_dataset['valid'].X valid_y = algorithm.monitoring_dataset['valid'].y y_pred = self.predictor.get_predictions(valid_x) threshold, score = self.symmetric_threshold.compute_optimal_threshold_and_score(valid_y, y_pred) print "Best threshold", threshold, '\ncorresponding F1Score:', score stat_dic = label_lists2types(valid_y, y_pred, sym_t=threshold) print values.TP, ':', stat_dic[values.TP], '\t\t', values.TN, ':', stat_dic[values.TN], '\n', \ values.FP, ':', stat_dic[values.FP], '\t\t', values.FN, ':', stat_dic[values.FN], '\n', \ values.FNP, ':', stat_dic[values.FNP], '\t\t', values.FNN, ':', stat_dic[values.FNN] for extension in self.call_list: # TODO: consider using inspect.getargspec() try: extension.on_monitor(model, dataset, algorithm, stat_dic) except TypeError: extension.on_monitor(model, dataset, algorithm)
def on_monitor(self, model, dataset, algorithm, stat_dic=None): import sys sys.path.append('..') from utils import values if stat_dic is None: from utils.casting import label_lists2types # obtaining validating set valid_x = algorithm.monitoring_dataset['valid'].X valid_y = algorithm.monitoring_dataset['valid'].y y_pred = self.predictor.get_predictions(valid_x) stat_dic = label_lists2types(valid_y, y_pred) # recall = TP/(TP + FN) if stat_dic[values.TP] == 0: recall = 0 else: recall = float(stat_dic[values.TP])/(stat_dic[values.TP] + stat_dic[values.FN]) self.recall_list.append(recall) print "recall:", recall
def on_monitor(self, model, dataset, algorithm, stat_dic=None): import sys sys.path.append('..') from utils import values if stat_dic is None: from utils.casting import label_lists2types # obtaining validating set valid_x = algorithm.monitoring_dataset['valid'].X valid_y = algorithm.monitoring_dataset['valid'].y y_pred = self.predictor.get_predictions(valid_x) stat_dic = label_lists2types(valid_y, y_pred) # accuracy = (TP + TN) / TOTAL if (stat_dic[values.TP] + stat_dic[values.TN]) == 0: accuracy = 0 else: accuracy = float(stat_dic[values.TP]+stat_dic[values.TN])/sum(stat_dic.values()) self.accuracy_list.append(accuracy) print "accuracy:", accuracy
def on_monitor(self, model, dataset, algorithm): import sys sys.path.append('..') from utils.casting import label_lists2types from utils import values print '\nSHOWING STATISTICS FOR NO THRESHOLD' valid_x = algorithm.monitoring_dataset['valid'].X valid_y = algorithm.monitoring_dataset['valid'].y y_pred = self.predictor.get_predictions(valid_x) stat_dic = label_lists2types(valid_y, y_pred) print values.TP, ':', stat_dic[values.TP], '\t\t', values.TN, ':', stat_dic[values.TN], '\n', \ values.FP, ':', stat_dic[values.FP], '\t\t', values.FN, ':', stat_dic[values.FN], '\n', \ values.FNP, ':', stat_dic[values.FNP], '\t\t', values.FNN, ':', stat_dic[values.FNN] for extension in self.call_list: # TODO: consider using inspect.getargspec() try: extension.on_monitor(model, dataset, algorithm, stat_dic) except TypeError: extension.on_monitor(model, dataset, algorithm)