def stats_from_proba(proba, Y_test):
# Binary and one output
if proba.shape[1] == 1:
proba = np.concatenate([1 - proba, proba], axis=1)
if len(Y_test.shape) == 1:
Y_test = np.transpose(np.array([1 - Y_test, Y_test]))
predicted_labels = np_utils.probas_to_classes(proba)
true_labels = np_utils.probas_to_classes(Y_test)
precision, recall, fbeta, support = sklearn.metrics.precision_recall_fscore_support(
predicted_labels, true_labels)
accuracy = sklearn.metrics.accuracy_score(predicted_labels, true_labels)
num_penalties, thresh_low, thresh_high = \
StatsUtils.yolo_oracle(Y_test[:, 1], proba[:, 1])
windowed_acc, windowed_supp = StatsUtils.windowed_accuracy(predicted_labels, Y_test)
metrics = {'precision': precision,
'recall': recall,
'fbeta': fbeta,
'support': support,
'accuracy': accuracy,
'penalities': num_penalties,
'windowed_accuracy': windowed_acc,
'windowed_support': windowed_supp}
return metrics
def get_data_with_split(csv_fname,
video_fname,
avg_fname,
num_frames=None,
start_frame=0,
OBJECTS=['car'],
resol=(50, 50),
center=True,
dtype='float32',
train_ratio=0.8):
def print_class_numbers(Y, nb_classes):
classes = np_utils.probas_to_classes(Y)
for i in range(nb_classes):
print('class %d: %d' % (i, np.sum(classes == i)))
print('Parsing %s, extracting %s' % (csv_fname, str(OBJECTS)))
all_counts = get_binary(csv_fname,
limit=num_frames,
OBJECTS=OBJECTS,
start=start_frame)
print('Retrieving all frames from %s' % video_fname)
all_frames = videoUtils.get_all_frames(len(all_counts),
video_fname,
scale=resol,
start=start_frame)
print('Splitting data into training and test sets')
X_train, X_test, Y_train, Y_test = to_test_train(avg_fname,
all_frames,
all_counts,
train_ratio=train_ratio)
nb_classes = all_counts.max() + 1
print(
'(train) positive examples: %d, total examples: %d' %
(np.count_nonzero(np_utils.probas_to_classes(Y_train)), len(Y_train)))
print_class_numbers(Y_train, nb_classes)
print('(test) positive examples: %d, total examples: %d' %
(np.count_nonzero(np_utils.probas_to_classes(Y_test)), len(Y_test)))
print_class_numbers(Y_test, nb_classes)
print('shape of image: ' + str(all_frames[0].shape))
print('number of classes: %d' % (nb_classes))
data = (X_train, Y_train, X_test, Y_test)
return data, nb_classes
def evaluate_model(model, X_test, Y_test, batch_size=256):
predicted_labels, test_time = get_labels(model, X_test, batch_size, True)
true_labels = np_utils.probas_to_classes(Y_test)
confusion = sklearn.metrics.confusion_matrix(true_labels, predicted_labels)
# Minor smoothing to prevent division by 0 errors
TN = float(confusion[0][0]) + 1
FN = float(confusion[1][0]) + 1
TP = float(confusion[1][1]) + 1
FP = float(confusion[0][1]) + 1
metrics = {'recall': TP / (TP + FN),
'specificity': TN / (FP + TN),
'precision': TP / (TP + FP),
'npv': TN / (TN + FN),
'fpr': FP / (FP + TN),
'fdr': FP / (FP + TP),
'fnr': FN / (FN + TP),
'accuracy': (TP + TN) / (TP + FP + TN + FN),
'f1': (2 * TP) / (2 * TP + FP + FN),
'test_time': test_time}
return metrics
def print_class_numbers(Y, nb_classes):
classes = np_utils.probas_to_classes(Y)
for i in range(nb_classes):
print('class %d: %d' % (i, np.sum(classes == i)))