def on_monitor(self, model, dataset, algorithm):
import sys
sys.path.append('..')
from utils.casting import label_lists2types
from utils import values
print '\nSHOWING STATISTICS FOR SYMMETRIC THRESHOLD'
valid_x = algorithm.monitoring_dataset['valid'].X
valid_y = algorithm.monitoring_dataset['valid'].y
y_pred = self.predictor.get_predictions(valid_x)
threshold, score = self.symmetric_threshold.compute_optimal_threshold_and_score(valid_y, y_pred)
print "Best threshold", threshold, '\ncorresponding F1Score:', score
stat_dic = label_lists2types(valid_y, y_pred, sym_t=threshold)
print values.TP, ':', stat_dic[values.TP], '\t\t', values.TN, ':', stat_dic[values.TN], '\n', \
values.FP, ':', stat_dic[values.FP], '\t\t', values.FN, ':', stat_dic[values.FN], '\n', \
values.FNP, ':', stat_dic[values.FNP], '\t\t', values.FNN, ':', stat_dic[values.FNN]
for extension in self.call_list:
# TODO: consider using inspect.getargspec()
try:
extension.on_monitor(model, dataset, algorithm, stat_dic)
except TypeError:
extension.on_monitor(model, dataset, algorithm)
def on_monitor(self, model, dataset, algorithm, stat_dic=None):
import sys
sys.path.append('..')
from utils import values
if stat_dic is None:
from utils.casting import label_lists2types
# obtaining validating set
valid_x = algorithm.monitoring_dataset['valid'].X
valid_y = algorithm.monitoring_dataset['valid'].y
y_pred = self.predictor.get_predictions(valid_x)
stat_dic = label_lists2types(valid_y, y_pred)
# recall = TP/(TP + FN)
if stat_dic[values.TP] == 0:
recall = 0
else:
recall = float(stat_dic[values.TP])/(stat_dic[values.TP] + stat_dic[values.FN])
self.recall_list.append(recall)
print "recall:", recall
def on_monitor(self, model, dataset, algorithm, stat_dic=None):
import sys
sys.path.append('..')
from utils import values
if stat_dic is None:
from utils.casting import label_lists2types
# obtaining validating set
valid_x = algorithm.monitoring_dataset['valid'].X
valid_y = algorithm.monitoring_dataset['valid'].y
y_pred = self.predictor.get_predictions(valid_x)
stat_dic = label_lists2types(valid_y, y_pred)
# accuracy = (TP + TN) / TOTAL
if (stat_dic[values.TP] + stat_dic[values.TN]) == 0:
accuracy = 0
else:
accuracy = float(stat_dic[values.TP]+stat_dic[values.TN])/sum(stat_dic.values())
self.accuracy_list.append(accuracy)
print "accuracy:", accuracy
def on_monitor(self, model, dataset, algorithm):
import sys
sys.path.append('..')
from utils.casting import label_lists2types
from utils import values
print '\nSHOWING STATISTICS FOR NO THRESHOLD'
valid_x = algorithm.monitoring_dataset['valid'].X
valid_y = algorithm.monitoring_dataset['valid'].y
y_pred = self.predictor.get_predictions(valid_x)
stat_dic = label_lists2types(valid_y, y_pred)
print values.TP, ':', stat_dic[values.TP], '\t\t', values.TN, ':', stat_dic[values.TN], '\n', \
values.FP, ':', stat_dic[values.FP], '\t\t', values.FN, ':', stat_dic[values.FN], '\n', \
values.FNP, ':', stat_dic[values.FNP], '\t\t', values.FNN, ':', stat_dic[values.FNN]
for extension in self.call_list:
# TODO: consider using inspect.getargspec()
try:
extension.on_monitor(model, dataset, algorithm, stat_dic)
except TypeError:
extension.on_monitor(model, dataset, algorithm)
