def test(args):
if not os.path.exists(os.path.join(DATASETS, args.dataset_name)):
print(f"Dataset {args.dataset_name} does not exist", file=sys.stderr)
sys.exit(1)
noise_classes = load_noise_classes(args)
# Extract features and test each classifier
filenames, classes, labels = read_dataset(args.dataset_name, "test")
if not args.skip_classifiers:
print("Testing individual classifiers")
for nc in noise_classes.values():
print(nc.name)
for spec in nc.classifiers:
feats, idxs = extract_features(filenames,
spec["feature"],
spec["feature_settings"],
spec.get("vad", None),
concatenate=False,
cache=not args.recompute)
label_ind, = np.where(
classes == spec["instance"].noise_types[0])[0]
labels_binary = labels[:, label_ind]
labels_binary = np.column_stack(
(labels_binary, labels_binary != 1))
print(spec["instance"].test(feats, idxs, labels_binary))
print()
# Stats for the noise class based labeling
args.filenames = filenames
predicted_labels, nc_ids = get_labels(args, noise_classes)
print("Combined classifier statistics")
confusion_tables, stats_dict = stats(predicted_labels, nc_ids, labels,
classes)
for nc_id, ct in confusion_tables.items():
print(noise_classes[nc_id].name)
if not args.skip_classifiers and len(
noise_classes[nc_id].classifiers) == 1:
print("(see above)")
print()
continue
print(ct)
print()
print("Multilabel statistics")
for name, value in stats_dict.items():
print(f"{name + ': ':<18}{value:.3}")
def _iterate_classifiers(spec_inds_sorted,
filenames,
recompute,
silent=False,
yield_ind=False):
last_feat_id = None
for spec_ind, nc in spec_inds_sorted:
spec = nc.classifiers[spec_ind]
if _feat_id(spec) != last_feat_id:
if not silent:
print(
f"Extracting {spec['feature']} features (use cache: {['yes', 'no'][recompute]})"
)
print(f"\tSettings: {spec['feature_settings']}")
print(f"\tVAD settings: {spec.get('vad', 'VAD not used')}")
feats, idxs = extract_features(filenames,
spec["feature"],
spec["feature_settings"],
spec.get("vad", None),
concatenate=False,
cache=not recompute)
last_feat_id = _feat_id(spec)
yield (spec_ind if yield_ind else spec), nc, feats, idxs
def error_analysis(collection, train_collection, background_model):
global l_avg_accum, r_avg_accum
train_data, train_labels, label_names, files = fe.generate_dataset(train_collection, background_model)
clf_l = Classifier('svc')
clf_r = Classifier('svc')
clf_l.train(train_data[0], train_labels[0])
clf_r.train(train_data[1], train_labels[1])
l_confusion_matrix = np.zeros((len(label_names), len(label_names)))
r_confusion_matrix = np.zeros((len(label_names), len(label_names)))
with open(collection, 'r') as f_collection:
print
print
print '***********************************'
print 'Error Analysis:', collection
print '***********************************'
# print '\t%s\t\t\t\t%s\t%s\t%s' % ('File Path', 'L', 'R', 'Actual')
l_errors = 0
r_errors = 0
n_samples = 0
sample_size = 10
# open each video in the collection
for line in f_collection:
a_line = line.replace('\n','').split('\t') # remove line breaks and split at tab
folder = a_line[0] # first element is folder containing video images
label = a_line[1] # next element is the ground truth label
# initialize hand detector
if label is '2':
hd = HandDetector('data/background_model_error2', folder) ### UPDATE: This is a hack for fixing some issues with error 2: will update after ICMC paper
else:
hd = HandDetector(background_model, folder)
right_sample = []
left_sample = []
i = 0
for left_hand, right_hand, filepath, img in hd.hand_generator():
if i<sample_size:
i += 1
right_sample.append(fe.extract_features(right_hand)) # extract the features
left_sample.append(fe.extract_features(left_hand))
else:
l_prediction = clf_l.majority_predict(np.array(left_sample))
r_prediction = clf_r.majority_predict(np.array(right_sample))
l_correct = l_prediction == int(label)
r_correct = r_prediction == int(label)
# if not (l_correct and r_correct):
# print '\t%s\t%s\t%s\t%s' % (filepath, l_prediction, r_prediction, label)
l_confusion_matrix[int(label)][l_prediction]+=1
r_confusion_matrix[int(label)][r_prediction]+=1
if not l_correct:
l_errors+=1
if not r_correct:
r_errors+=1
# Reset sample and populate with current hands
right_sample = []
left_sample = []
right_sample.append(fe.extract_features(right_hand))
left_sample.append(fe.extract_features(left_hand))
i = 1
n_samples+=1
print
print 'Error Rate:', collection
print 'Left:', (l_errors / float(n_samples))
print 'Right:', (r_errors / float(n_samples))
print
print
l_avg_accum += (l_errors / float(n_samples))
r_avg_accum += (r_errors / float(n_samples))
def error_analysis(collection, hd, clf_left, clf_right, label_names, file_ext, name, display=False):
clf_l = clf_left
clf_r = clf_right
with open(collection, 'r') as f_collection:
print
print
print '***********************************'
print 'Error Analysis:', collection
print '***********************************'
# print '\t%s\t\t\t\t%s\t%s\t%s' % ('File Path', 'L', 'R', 'Actual')
l_confusion_matrix = np.zeros((len(label_names), len(label_names)))
r_confusion_matrix = np.zeros((len(label_names), len(label_names)))
l_errors = 0
r_errors = 0
samples = 0
# open each video in the collection
for line in f_collection:
a_line = line.replace('\n','').split('\t') # remove line breaks and split at tab
folder = a_line[0] # first element is folder containing video images
background = a_line[1]
label = a_line[2] # next element is the ground truth label
# initialize hand detector
hd.initialize_detector(background, folder, file_ext)
for hand in hd.hand_generator():
left_hand, right_hand = hand.get_hands()
try:
right_features = fe.extract_features(right_hand) # extract the features
left_features = fe.extract_features(left_hand)
l_prediction = clf_l.predict(left_features)[0]
r_prediction = clf_r.predict(right_features)[0]
hand.set_label( (int(label), int(label)) )
hand.set_predicted( (l_prediction, r_prediction) )
l_correct = l_prediction == int(label)
r_correct = r_prediction == int(label)
# if not (l_correct and r_correct):
# # print '\t%s\t%s\t%s\t%s' % (filepath, l_prediction, r_prediction, label)
# for sv in
l_confusion_matrix[int(label)][l_prediction]+=1
r_confusion_matrix[int(label)][r_prediction]+=1
if not l_correct:
l_errors+=1
if not r_correct:
r_errors+=1
if display:
hand.show()
samples+=1
except:
print 'Issue with:', hand.get_filepath()
# cv2.destroyAllWindows()
print
print 'Collection:', collection
print line
print 'Left Hand'
print '##########################'
print
print 'Accuracy:', 1-(l_errors / float(samples))
print
print '#### Confusion Matrix'
print_confusion_matrix(l_confusion_matrix)
print
# print 'Precision and Recall'
# print_precision_recall(l_confusion_matrix, label_names)
print
print
print 'Right Hand Confusion Matrix'
print '###########################'
print
print 'Accuracy:', 1-(r_errors / float(samples))
print
print '#### Confusion Matrix'
print_confusion_matrix(r_confusion_matrix)
print
# print 'Precision and Recall'
# print_precision_recall(r_confusion_matrix, label_names)
print
print
print "#### Combined Confusion Matrix"
print_confusion_matrix(r_confusion_matrix + l_confusion_matrix)
# plot_confusion_matrix(r_confusion_matrix + l_confusion_matrix, title='Combined Hand Confusion Matrix', cmap='Greens')
print
