def eval_roc_and_pr(argv): args = parse_arguments(argv) print 'args:', args # for k,v in args: # print "{}: {}".format(k,v) save_dir = args.save_dir if not osp.exists(save_dir): os.makedirs(save_dir) fp_log = open(osp.join(save_dir, 'eval.log'), 'w') fp_log.write('args:\n{}\n\n'.format(args)) do_norm = not (args.no_normalize) ftr_mat = load_mat_features(args.feature_mat_file, do_norm) gallery_img_list, gallery_idx_list, gallery_id_list = load_image_list( args.gallery_list_file) probe_img_list, probe_idx_list, probe_id_list = load_image_list( args.probe_list_file) # gallery_img_list, gallery_idx_list, gallery_id_list = load_image_list( # args.gallery_list_file, 100) # probe_img_list, probe_idx_list, probe_id_list = load_image_list( # args.probe_list_file, 100) gallery_ftrs = ftr_mat[np.array(gallery_idx_list)] print "gallery_ftrs.shape: ", gallery_ftrs.shape probe_ftrs = ftr_mat[np.array(probe_idx_list)] print "probe_ftrs.shape: ", probe_ftrs.shape n_gallery = len(gallery_img_list) n_probe = len(probe_img_list) gt_mat = generate_gt_mat(probe_id_list, gallery_id_list) # similarity_mat = np.dot(probe_ftrs, gallery_ftrs.T) similarity_mat = calc_similarity_mat(probe_ftrs, gallery_ftrs) print similarity_mat pos_pairs = gt_mat.sum() neg_pairs = gt_mat.size - pos_pairs fp_log.write('Total pairs: {}\n'.format(n_gallery * n_probe)) fp_log.write('Effective pos pairs: {}\n'.format(pos_pairs)) fp_log.write('Effective neg pairs: {}\n'.format(neg_pairs)) fp_log.close() threshs = None # calc_roc_local(similarity_mat, gt_mat, threshs) tp, fn, tn, fp = calc_roc(similarity_mat, gt_mat, threshs, save_dir) calc_presicion_recall(tp, fn, tn, fp, threshs, save_dir) draw_analysis_figure(tp, fn, tn, fp, save_dir, True)
def eval_roc_and_pr(argv): args = parse_arguments(argv) print 'args:', args # for k,v in args: # print "{}: {}".format(k,v) save_dir = args.save_dir if not osp.exists(save_dir): os.makedirs(save_dir) fp_log = open(osp.join(save_dir, 'eval.log'), 'w') fp_log.write('args:\n{}\n\n'.format(args)) fp_log.close() do_norm = not (args.no_normalize) ftr_mat = load_mat_features(args.feature_mat_file, do_norm) same_pairs_idx_list, same_img_list = load_image_pairs(args.same_pairs_file) same_sim_list = calc_similarity(ftr_mat, same_pairs_idx_list) fn_same = osp.join(save_dir, 'same_pairs_similarity.txt') fp_same = open(fn_same, 'w') for (i, sim) in enumerate(same_sim_list): fp_same.write("%s %s %f\n" % (same_img_list[i][0], same_img_list[i][1], sim)) fp_same.close() diff_pairs_idx_list, diff_img_list = load_image_pairs(args.diff_pairs_file) diff_sim_list = calc_similarity(ftr_mat, diff_pairs_idx_list) fn_diff = osp.join(save_dir, 'diff_pairs_similarity.txt') fp_diff = open(fn_diff, 'w') for (i, sim) in enumerate(diff_sim_list): fp_diff.write("%s %s %f\n" % (diff_img_list[i][0], diff_img_list[i][1], sim)) fp_diff.close() threshs = None tp, fn, tn, fp = calc_roc(same_sim_list, diff_sim_list, threshs, save_dir) calc_presicion_recall(tp, fn, tn, fp, threshs, save_dir) draw_analysis_figure(tp, fn, tn, fp, save_dir, True)
def eval_roc_and_pr(argv): args = parse_arguments(argv) print 'args:', args # for k,v in args: # print "{}: {}".format(k,v) save_dir = args.save_dir if not osp.exists(save_dir): os.makedirs(save_dir) fp_log = open(osp.join(save_dir, 'eval.log'), 'w') fp_log.write('args:\n{}\n\n'.format(args)) do_norm = not(args.no_normalize) img_list, id_list = load_image_list(args.image_list_file) # ftr_mat = load_mat_features(args.feature_mat_file, do_norm)[:100] # gt_mat = generate_gt_mat(id_list[:100]) ftr_mat = load_mat_features(args.feature_mat_file, do_norm) gt_mat = generate_gt_mat(id_list) num_imgs = len(img_list) pos_pairs = gt_mat.sum() neg_pairs = num_imgs * num_imgs - pos_pairs pos_pairs -= num_imgs fp_log.write('Effective pos pairs: {}\n'.format(pos_pairs*0.5)) fp_log.write('Effective neg pairs: {}\n'.format(neg_pairs*0.5)) fp_log.close() similarity_mat = calc_similarity_mat(ftr_mat) print similarity_mat threshs = None # calc_roc_local(similarity_mat, gt_mat, threshs) tp, fn, tn, fp = calc_roc(similarity_mat, gt_mat, threshs, save_dir) calc_presicion_recall(tp, fn, tn, fp, threshs, save_dir) draw_analysis_figure(tp, fn, tn, fp, save_dir, True)
def main(feat_mat_file, save_dir=None): if feat_mat_file.endswith('.mat'): base_fn = osp.splitext(feat_mat_file)[0] else: base_fn = feat_mat_file feat_mat_file = base_fn + '.mat' if not osp.exists(feat_mat_file): print 'File not exists: ', feat_mat_file avg_mat_fn = base_fn + '_mirror_eltavg.mat' if not osp.exists(avg_mat_fn): print 'File not exists: ', avg_mat_fn if save_dir and not osp.isdir(save_dir): os.makedirs(save_dir) orig_ftr_mat = load_mat_features(feat_mat_file, False) avg_ftr_mat = load_mat_features(avg_mat_fn, False) print 'orignal feat mat shape: ', orig_ftr_mat.shape print 'average feat mat shape: ', avg_ftr_mat.shape flip_mat_fn = base_fn + '_flip.mat' if save_dir: flip_mat_fn = osp.join(save_dir, osp.basename(flip_mat_fn)) flip_ftr_mat = avg_ftr_mat * 2.0 - orig_ftr_mat print 'flipped feat mat shape: ', flip_ftr_mat.shape sio.savemat(flip_mat_fn, {'features': flip_ftr_mat}) if TEST_FEAT_SIM: for i in range(10): sim = calc_similarity(orig_ftr_mat[i], flip_ftr_mat[i]) print '===> %d pair of (orignal, flipped) featur: ' % i print 'sim = ', sim # concat original feature and flip feature concat_mat_fn = base_fn + '_concat.mat' if save_dir: concat_mat_fn = osp.join(save_dir, osp.basename(concat_mat_fn)) concat_ftr_mat = np.hstack((orig_ftr_mat, flip_ftr_mat)) print 'concat feat mat shape: ', concat_ftr_mat.shape sio.savemat(concat_mat_fn, {'features': concat_ftr_mat}) # normalize features orig_ftr_mat = normalize_features(orig_ftr_mat) flip_ftr_mat = normalize_features(flip_ftr_mat) # concat normalized original feature and flip feature concat_norm_mat_fn = base_fn + '_concat_norm.mat' if save_dir: concat_norm_mat_fn = osp.join(save_dir, osp.basename(concat_norm_mat_fn)) concat_norm_ftr_mat = np.hstack((orig_ftr_mat, flip_ftr_mat)) print 'concat norm feat mat shape: ', concat_norm_ftr_mat.shape sio.savemat(concat_norm_mat_fn, {'features': concat_norm_ftr_mat}) # calc average of normalized original feature and flip feature avg_norm_mat_fn = base_fn + '_avg_norm.mat' if save_dir: avg_norm_mat_fn = osp.join(save_dir, osp.basename(avg_norm_mat_fn)) avg_norm_ftr_mat = (orig_ftr_mat + flip_ftr_mat) * 0.5 print 'avg norm feat mat shape: ', avg_norm_ftr_mat.shape sio.savemat(avg_norm_mat_fn, {'features': avg_norm_ftr_mat})
def main(args): print 'args:', args # for k,v in args: # print "{}: {}".format(k,v) ftr_mat, actual_issame = load_mat_features(args.feature_mat_file, args.lfw_pairs_mat_file, args.lfw_nrof_folds, args.nrof_features_per_fold) print('ftr_mat.shape: {}'.format(ftr_mat.shape)) print('actual_issame.shape: {}'.format(actual_issame.shape)) print('\n=======================================') print('EVALUATION WITHOUT K-FOLDS') tpr, fpr, best_accuracy, val_far = lfw_evaluate(ftr_mat, actual_issame, distance=args.distance) if args.draw_roc: plt.figure() plt.plot(fpr, tpr) plt.title('ROC without K-folds') plt.show() print('Accuracy: %2.5f @distance_threshold=%2.5f, VAL=%2.5f, FAR=%2.5f' % (best_accuracy['accuracy'], best_accuracy['threshold'], best_accuracy['VAL'], best_accuracy['FAR'])) auc = get_auc(fpr, tpr) print('Area Under Curve (AUC): %2.5f' % auc) eer = get_eer(fpr, tpr) print('Equal Error Rate (EER): %2.5f with accuracy=%2.5f' % (eer, 1.0 - eer)) for it in val_far: print('Validation rate: %2.5f @ FAR=%2.5f with theshold %2.5f' % (it['VAL'], it['FAR'], it['threshold'])) print('\n=======================================') print('EVALUATION WITH K-FOLDS') tpr, fpr, accuracy, val, val_std, far = lfw_evaluate_kfolds( ftr_mat, actual_issame, distance=args.distance, nrof_folds=args.lfw_nrof_folds) if args.draw_roc: plt.figure() plt.plot(fpr, tpr) plt.title('ROC with K-folds') plt.show() print('Accuracy: %2.5f+-%2.5f' % (np.mean(accuracy), np.std(accuracy))) print('Validation rate: %2.5f+-%2.5f @ FAR=%2.5f' % (val, val_std, far)) auc = get_auc(fpr, tpr) print('Area Under Curve (AUC): %2.5f' % auc) eer = get_eer(fpr, tpr) print('Equal Error Rate (EER): %2.5f with accuracy=%2.5f' % (eer, 1.0 - eer))
def eval_topN(argv): args = parse_arguments(argv) print 'args:', args # for k,v in args: # print "{}: {}".format(k,v) top_N = args.top_n if top_N < 5: top_N = 5 save_dir = args.save_dir if not osp.exists(save_dir): os.makedirs(save_dir) fn_rlt = osp.join(save_dir, 'ident_test_topN_details.txt') fn_rlt2 = osp.join(save_dir, 'ident_test_topN_rlt.txt') do_norm = not (args.no_normalize) ftr_mat = load_mat_features(args.feature_mat_file, do_norm) gallery_img_list, gallery_idx_list, gallery_id_list = load_image_list( args.gallery_list_file) gallery_ftrs = ftr_mat[np.array(gallery_idx_list)] print "gallery_ftrs.shape: ", gallery_ftrs.shape n_gallery = len(gallery_img_list) if top_N > n_gallery: top_N = n_gallery probe_img_list, probe_idx_list, probe_id_list = load_image_list( args.probe_list_file) probe_ftrs = ftr_mat[np.array(probe_idx_list)] print "probe_ftrs.shape: ", probe_ftrs.shape n_probe = len(probe_img_list) similarity_mat = np.dot(probe_ftrs, gallery_ftrs.T) print "similarity_mat.shape: ", similarity_mat.shape fp_rlt = open(fn_rlt, 'w') top_N_cnt_ttl = np.zeros(top_N) for i in range(n_probe): top_N_cnt = np.zeros(top_N) sim_row = similarity_mat[i] top_N_idx_idx = np.argsort(-sim_row)[:top_N + 1] top_one_idx = gallery_idx_list[top_N_idx_idx[0]] # if i==0: # print "probe img_name: ", probe_img_list[i] # print "probe img_idx: ", probe_idx_list[i] # print "probe img_id: ", probe_id_list[i] # # print 'sim_row.max: ', sim_row.max() # # print "top_1 img_name: ", gallery_img_list[top_N_idx_idx[0]] # # print "===>Before check top_1's img_idx\n" # print "top_N_idx_idx:", top_N_idx_idx # print "top_N sim: ", sim_row[top_N_idx_idx] # print "top_N_idx:", gallery_idx_list[top_N_idx_idx] # print "top_N_id:", gallery_id_list[top_N_idx_idx] if top_one_idx == probe_idx_list[i]: top_N_idx_idx = top_N_idx_idx[1:top_N + 1] else: top_N_idx_idx = top_N_idx_idx[0:top_N] # if i==0: # print "===>After check top_1's img_idx\n" # print "top_N_idx_idx:", top_N_idx_idx # print "top_N sim: ", sim_row[top_N_idx_idx] # print "top_N_idx:", gallery_idx_list[top_N_idx_idx] # print "top_N_id:", gallery_id_list[top_N_idx_idx] for j, idx_idx in enumerate(top_N_idx_idx): top_N_cnt[j] += (probe_id_list[i] == gallery_id_list[idx_idx]) # if i==0: # print "top_N_cnt:", top_N_cnt for j in range(1, top_N): top_N_cnt[j] += top_N_cnt[j - 1] # # if i==0: # print "top_N_cnt:", top_N_cnt fp_rlt.write("%s" % probe_img_list[i]) for j in range(top_N): fp_rlt.write("\t%5d" % top_N_cnt[j]) top_N_cnt_ttl[j] += (top_N_cnt[j] > 0) fp_rlt.write("\n") fp_rlt.close() fp_rlt = open(fn_rlt2, 'w') fp_rlt.write("args: \n{}\n\n".format(args)) fp_rlt.write("Num of gallery features: %d\n" % n_gallery) fp_rlt.write("Num of probe features: %d\n\n" % n_probe) top_N_ratio = top_N_cnt_ttl / n_probe print "top_N_cnt_ttl: ", top_N_cnt_ttl print "top_N_ratio: ", top_N_ratio fp_rlt.write("TOP_N \tcnt \t ratio\n") fp_rlt.write("----------------------\n") for j in range(top_N): fp_rlt.write("top_%d\t%5d\t%5.4f\n" % (j + 1, top_N_cnt_ttl[j], top_N_ratio[j])) fp_rlt.write("\n") # for j in range(top_N): # fp_rlt.write("%5d\t" % top_N_cnt_ttl[j]) # fp_rlt.write("\n") # # for j in range(top_N): # fp_rlt.write("%5d\t" % top_N_ratio[j]) fp_rlt.write("\n") fp_rlt.close()