def test(args): if not os.path.exists(os.path.join(DATASETS, args.dataset_name)): print(f"Dataset {args.dataset_name} does not exist", file=sys.stderr) sys.exit(1) noise_classes = load_noise_classes(args) # Extract features and test each classifier filenames, classes, labels = read_dataset(args.dataset_name, "test") if not args.skip_classifiers: print("Testing individual classifiers") for nc in noise_classes.values(): print(nc.name) for spec in nc.classifiers: feats, idxs = extract_features(filenames, spec["feature"], spec["feature_settings"], spec.get("vad", None), concatenate=False, cache=not args.recompute) label_ind, = np.where( classes == spec["instance"].noise_types[0])[0] labels_binary = labels[:, label_ind] labels_binary = np.column_stack( (labels_binary, labels_binary != 1)) print(spec["instance"].test(feats, idxs, labels_binary)) print() # Stats for the noise class based labeling args.filenames = filenames predicted_labels, nc_ids = get_labels(args, noise_classes) print("Combined classifier statistics") confusion_tables, stats_dict = stats(predicted_labels, nc_ids, labels, classes) for nc_id, ct in confusion_tables.items(): print(noise_classes[nc_id].name) if not args.skip_classifiers and len( noise_classes[nc_id].classifiers) == 1: print("(see above)") print() continue print(ct) print() print("Multilabel statistics") for name, value in stats_dict.items(): print(f"{name + ': ':<18}{value:.3}")
def _iterate_classifiers(spec_inds_sorted, filenames, recompute, silent=False, yield_ind=False): last_feat_id = None for spec_ind, nc in spec_inds_sorted: spec = nc.classifiers[spec_ind] if _feat_id(spec) != last_feat_id: if not silent: print( f"Extracting {spec['feature']} features (use cache: {['yes', 'no'][recompute]})" ) print(f"\tSettings: {spec['feature_settings']}") print(f"\tVAD settings: {spec.get('vad', 'VAD not used')}") feats, idxs = extract_features(filenames, spec["feature"], spec["feature_settings"], spec.get("vad", None), concatenate=False, cache=not recompute) last_feat_id = _feat_id(spec) yield (spec_ind if yield_ind else spec), nc, feats, idxs
def error_analysis(collection, train_collection, background_model): global l_avg_accum, r_avg_accum train_data, train_labels, label_names, files = fe.generate_dataset(train_collection, background_model) clf_l = Classifier('svc') clf_r = Classifier('svc') clf_l.train(train_data[0], train_labels[0]) clf_r.train(train_data[1], train_labels[1]) l_confusion_matrix = np.zeros((len(label_names), len(label_names))) r_confusion_matrix = np.zeros((len(label_names), len(label_names))) with open(collection, 'r') as f_collection: print print print '***********************************' print 'Error Analysis:', collection print '***********************************' # print '\t%s\t\t\t\t%s\t%s\t%s' % ('File Path', 'L', 'R', 'Actual') l_errors = 0 r_errors = 0 n_samples = 0 sample_size = 10 # open each video in the collection for line in f_collection: a_line = line.replace('\n','').split('\t') # remove line breaks and split at tab folder = a_line[0] # first element is folder containing video images label = a_line[1] # next element is the ground truth label # initialize hand detector if label is '2': hd = HandDetector('data/background_model_error2', folder) ### UPDATE: This is a hack for fixing some issues with error 2: will update after ICMC paper else: hd = HandDetector(background_model, folder) right_sample = [] left_sample = [] i = 0 for left_hand, right_hand, filepath, img in hd.hand_generator(): if i<sample_size: i += 1 right_sample.append(fe.extract_features(right_hand)) # extract the features left_sample.append(fe.extract_features(left_hand)) else: l_prediction = clf_l.majority_predict(np.array(left_sample)) r_prediction = clf_r.majority_predict(np.array(right_sample)) l_correct = l_prediction == int(label) r_correct = r_prediction == int(label) # if not (l_correct and r_correct): # print '\t%s\t%s\t%s\t%s' % (filepath, l_prediction, r_prediction, label) l_confusion_matrix[int(label)][l_prediction]+=1 r_confusion_matrix[int(label)][r_prediction]+=1 if not l_correct: l_errors+=1 if not r_correct: r_errors+=1 # Reset sample and populate with current hands right_sample = [] left_sample = [] right_sample.append(fe.extract_features(right_hand)) left_sample.append(fe.extract_features(left_hand)) i = 1 n_samples+=1 print print 'Error Rate:', collection print 'Left:', (l_errors / float(n_samples)) print 'Right:', (r_errors / float(n_samples)) print print l_avg_accum += (l_errors / float(n_samples)) r_avg_accum += (r_errors / float(n_samples))
def error_analysis(collection, hd, clf_left, clf_right, label_names, file_ext, name, display=False): clf_l = clf_left clf_r = clf_right with open(collection, 'r') as f_collection: print print print '***********************************' print 'Error Analysis:', collection print '***********************************' # print '\t%s\t\t\t\t%s\t%s\t%s' % ('File Path', 'L', 'R', 'Actual') l_confusion_matrix = np.zeros((len(label_names), len(label_names))) r_confusion_matrix = np.zeros((len(label_names), len(label_names))) l_errors = 0 r_errors = 0 samples = 0 # open each video in the collection for line in f_collection: a_line = line.replace('\n','').split('\t') # remove line breaks and split at tab folder = a_line[0] # first element is folder containing video images background = a_line[1] label = a_line[2] # next element is the ground truth label # initialize hand detector hd.initialize_detector(background, folder, file_ext) for hand in hd.hand_generator(): left_hand, right_hand = hand.get_hands() try: right_features = fe.extract_features(right_hand) # extract the features left_features = fe.extract_features(left_hand) l_prediction = clf_l.predict(left_features)[0] r_prediction = clf_r.predict(right_features)[0] hand.set_label( (int(label), int(label)) ) hand.set_predicted( (l_prediction, r_prediction) ) l_correct = l_prediction == int(label) r_correct = r_prediction == int(label) # if not (l_correct and r_correct): # # print '\t%s\t%s\t%s\t%s' % (filepath, l_prediction, r_prediction, label) # for sv in l_confusion_matrix[int(label)][l_prediction]+=1 r_confusion_matrix[int(label)][r_prediction]+=1 if not l_correct: l_errors+=1 if not r_correct: r_errors+=1 if display: hand.show() samples+=1 except: print 'Issue with:', hand.get_filepath() # cv2.destroyAllWindows() print print 'Collection:', collection print line print 'Left Hand' print '##########################' print print 'Accuracy:', 1-(l_errors / float(samples)) print print '#### Confusion Matrix' print_confusion_matrix(l_confusion_matrix) print # print 'Precision and Recall' # print_precision_recall(l_confusion_matrix, label_names) print print print 'Right Hand Confusion Matrix' print '###########################' print print 'Accuracy:', 1-(r_errors / float(samples)) print print '#### Confusion Matrix' print_confusion_matrix(r_confusion_matrix) print # print 'Precision and Recall' # print_precision_recall(r_confusion_matrix, label_names) print print print "#### Combined Confusion Matrix" print_confusion_matrix(r_confusion_matrix + l_confusion_matrix) # plot_confusion_matrix(r_confusion_matrix + l_confusion_matrix, title='Combined Hand Confusion Matrix', cmap='Greens') print