def eval_roc_and_pr(argv):
args = parse_arguments(argv)
print 'args:', args
# for k,v in args:
# print "{}: {}".format(k,v)
save_dir = args.save_dir
if not osp.exists(save_dir):
os.makedirs(save_dir)
fp_log = open(osp.join(save_dir, 'eval.log'), 'w')
fp_log.write('args:\n{}\n\n'.format(args))
do_norm = not (args.no_normalize)
ftr_mat = load_mat_features(args.feature_mat_file, do_norm)
gallery_img_list, gallery_idx_list, gallery_id_list = load_image_list(
args.gallery_list_file)
probe_img_list, probe_idx_list, probe_id_list = load_image_list(
args.probe_list_file)
# gallery_img_list, gallery_idx_list, gallery_id_list = load_image_list(
# args.gallery_list_file, 100)
# probe_img_list, probe_idx_list, probe_id_list = load_image_list(
# args.probe_list_file, 100)
gallery_ftrs = ftr_mat[np.array(gallery_idx_list)]
print "gallery_ftrs.shape: ", gallery_ftrs.shape
probe_ftrs = ftr_mat[np.array(probe_idx_list)]
print "probe_ftrs.shape: ", probe_ftrs.shape
n_gallery = len(gallery_img_list)
n_probe = len(probe_img_list)
gt_mat = generate_gt_mat(probe_id_list, gallery_id_list)
# similarity_mat = np.dot(probe_ftrs, gallery_ftrs.T)
similarity_mat = calc_similarity_mat(probe_ftrs, gallery_ftrs)
print similarity_mat
pos_pairs = gt_mat.sum()
neg_pairs = gt_mat.size - pos_pairs
fp_log.write('Total pairs: {}\n'.format(n_gallery * n_probe))
fp_log.write('Effective pos pairs: {}\n'.format(pos_pairs))
fp_log.write('Effective neg pairs: {}\n'.format(neg_pairs))
fp_log.close()
threshs = None
# calc_roc_local(similarity_mat, gt_mat, threshs)
tp, fn, tn, fp = calc_roc(similarity_mat, gt_mat, threshs, save_dir)
calc_presicion_recall(tp, fn, tn, fp, threshs, save_dir)
draw_analysis_figure(tp, fn, tn, fp, save_dir, True)
def eval_roc_and_pr(argv):
args = parse_arguments(argv)
print 'args:', args
# for k,v in args:
# print "{}: {}".format(k,v)
save_dir = args.save_dir
if not osp.exists(save_dir):
os.makedirs(save_dir)
fp_log = open(osp.join(save_dir, 'eval.log'), 'w')
fp_log.write('args:\n{}\n\n'.format(args))
fp_log.close()
do_norm = not (args.no_normalize)
ftr_mat = load_mat_features(args.feature_mat_file, do_norm)
same_pairs_idx_list, same_img_list = load_image_pairs(args.same_pairs_file)
same_sim_list = calc_similarity(ftr_mat, same_pairs_idx_list)
fn_same = osp.join(save_dir, 'same_pairs_similarity.txt')
fp_same = open(fn_same, 'w')
for (i, sim) in enumerate(same_sim_list):
fp_same.write("%s %s %f\n" %
(same_img_list[i][0], same_img_list[i][1], sim))
fp_same.close()
diff_pairs_idx_list, diff_img_list = load_image_pairs(args.diff_pairs_file)
diff_sim_list = calc_similarity(ftr_mat, diff_pairs_idx_list)
fn_diff = osp.join(save_dir, 'diff_pairs_similarity.txt')
fp_diff = open(fn_diff, 'w')
for (i, sim) in enumerate(diff_sim_list):
fp_diff.write("%s %s %f\n" %
(diff_img_list[i][0], diff_img_list[i][1], sim))
fp_diff.close()
threshs = None
tp, fn, tn, fp = calc_roc(same_sim_list, diff_sim_list, threshs, save_dir)
calc_presicion_recall(tp, fn, tn, fp, threshs, save_dir)
draw_analysis_figure(tp, fn, tn, fp, save_dir, True)
def eval_roc_and_pr(argv):
args = parse_arguments(argv)
print 'args:', args
# for k,v in args:
# print "{}: {}".format(k,v)
save_dir = args.save_dir
if not osp.exists(save_dir):
os.makedirs(save_dir)
fp_log = open(osp.join(save_dir, 'eval.log'), 'w')
fp_log.write('args:\n{}\n\n'.format(args))
do_norm = not(args.no_normalize)
img_list, id_list = load_image_list(args.image_list_file)
# ftr_mat = load_mat_features(args.feature_mat_file, do_norm)[:100]
# gt_mat = generate_gt_mat(id_list[:100])
ftr_mat = load_mat_features(args.feature_mat_file, do_norm)
gt_mat = generate_gt_mat(id_list)
num_imgs = len(img_list)
pos_pairs = gt_mat.sum()
neg_pairs = num_imgs * num_imgs - pos_pairs
pos_pairs -= num_imgs
fp_log.write('Effective pos pairs: {}\n'.format(pos_pairs*0.5))
fp_log.write('Effective neg pairs: {}\n'.format(neg_pairs*0.5))
fp_log.close()
similarity_mat = calc_similarity_mat(ftr_mat)
print similarity_mat
threshs = None
# calc_roc_local(similarity_mat, gt_mat, threshs)
tp, fn, tn, fp = calc_roc(similarity_mat, gt_mat, threshs, save_dir)
calc_presicion_recall(tp, fn, tn, fp, threshs, save_dir)
draw_analysis_figure(tp, fn, tn, fp, save_dir, True)
def main(feat_mat_file, save_dir=None):
if feat_mat_file.endswith('.mat'):
base_fn = osp.splitext(feat_mat_file)[0]
else:
base_fn = feat_mat_file
feat_mat_file = base_fn + '.mat'
if not osp.exists(feat_mat_file):
print 'File not exists: ', feat_mat_file
avg_mat_fn = base_fn + '_mirror_eltavg.mat'
if not osp.exists(avg_mat_fn):
print 'File not exists: ', avg_mat_fn
if save_dir and not osp.isdir(save_dir):
os.makedirs(save_dir)
orig_ftr_mat = load_mat_features(feat_mat_file, False)
avg_ftr_mat = load_mat_features(avg_mat_fn, False)
print 'orignal feat mat shape: ', orig_ftr_mat.shape
print 'average feat mat shape: ', avg_ftr_mat.shape
flip_mat_fn = base_fn + '_flip.mat'
if save_dir:
flip_mat_fn = osp.join(save_dir, osp.basename(flip_mat_fn))
flip_ftr_mat = avg_ftr_mat * 2.0 - orig_ftr_mat
print 'flipped feat mat shape: ', flip_ftr_mat.shape
sio.savemat(flip_mat_fn, {'features': flip_ftr_mat})
if TEST_FEAT_SIM:
for i in range(10):
sim = calc_similarity(orig_ftr_mat[i], flip_ftr_mat[i])
print '===> %d pair of (orignal, flipped) featur: ' % i
print 'sim = ', sim
# concat original feature and flip feature
concat_mat_fn = base_fn + '_concat.mat'
if save_dir:
concat_mat_fn = osp.join(save_dir, osp.basename(concat_mat_fn))
concat_ftr_mat = np.hstack((orig_ftr_mat, flip_ftr_mat))
print 'concat feat mat shape: ', concat_ftr_mat.shape
sio.savemat(concat_mat_fn, {'features': concat_ftr_mat})
# normalize features
orig_ftr_mat = normalize_features(orig_ftr_mat)
flip_ftr_mat = normalize_features(flip_ftr_mat)
# concat normalized original feature and flip feature
concat_norm_mat_fn = base_fn + '_concat_norm.mat'
if save_dir:
concat_norm_mat_fn = osp.join(save_dir,
osp.basename(concat_norm_mat_fn))
concat_norm_ftr_mat = np.hstack((orig_ftr_mat, flip_ftr_mat))
print 'concat norm feat mat shape: ', concat_norm_ftr_mat.shape
sio.savemat(concat_norm_mat_fn, {'features': concat_norm_ftr_mat})
# calc average of normalized original feature and flip feature
avg_norm_mat_fn = base_fn + '_avg_norm.mat'
if save_dir:
avg_norm_mat_fn = osp.join(save_dir, osp.basename(avg_norm_mat_fn))
avg_norm_ftr_mat = (orig_ftr_mat + flip_ftr_mat) * 0.5
print 'avg norm feat mat shape: ', avg_norm_ftr_mat.shape
sio.savemat(avg_norm_mat_fn, {'features': avg_norm_ftr_mat})
def main(args):
print 'args:', args
# for k,v in args:
# print "{}: {}".format(k,v)
ftr_mat, actual_issame = load_mat_features(args.feature_mat_file,
args.lfw_pairs_mat_file,
args.lfw_nrof_folds,
args.nrof_features_per_fold)
print('ftr_mat.shape: {}'.format(ftr_mat.shape))
print('actual_issame.shape: {}'.format(actual_issame.shape))
print('\n=======================================')
print('EVALUATION WITHOUT K-FOLDS')
tpr, fpr, best_accuracy, val_far = lfw_evaluate(ftr_mat,
actual_issame,
distance=args.distance)
if args.draw_roc:
plt.figure()
plt.plot(fpr, tpr)
plt.title('ROC without K-folds')
plt.show()
print('Accuracy: %2.5f @distance_threshold=%2.5f, VAL=%2.5f, FAR=%2.5f' %
(best_accuracy['accuracy'], best_accuracy['threshold'],
best_accuracy['VAL'], best_accuracy['FAR']))
auc = get_auc(fpr, tpr)
print('Area Under Curve (AUC): %2.5f' % auc)
eer = get_eer(fpr, tpr)
print('Equal Error Rate (EER): %2.5f with accuracy=%2.5f' %
(eer, 1.0 - eer))
for it in val_far:
print('Validation rate: %2.5f @ FAR=%2.5f with theshold %2.5f' %
(it['VAL'], it['FAR'], it['threshold']))
print('\n=======================================')
print('EVALUATION WITH K-FOLDS')
tpr, fpr, accuracy, val, val_std, far = lfw_evaluate_kfolds(
ftr_mat,
actual_issame,
distance=args.distance,
nrof_folds=args.lfw_nrof_folds)
if args.draw_roc:
plt.figure()
plt.plot(fpr, tpr)
plt.title('ROC with K-folds')
plt.show()
print('Accuracy: %2.5f+-%2.5f' % (np.mean(accuracy), np.std(accuracy)))
print('Validation rate: %2.5f+-%2.5f @ FAR=%2.5f' % (val, val_std, far))
auc = get_auc(fpr, tpr)
print('Area Under Curve (AUC): %2.5f' % auc)
eer = get_eer(fpr, tpr)
print('Equal Error Rate (EER): %2.5f with accuracy=%2.5f' %
(eer, 1.0 - eer))
def eval_topN(argv):
args = parse_arguments(argv)
print 'args:', args
# for k,v in args:
# print "{}: {}".format(k,v)
top_N = args.top_n
if top_N < 5:
top_N = 5
save_dir = args.save_dir
if not osp.exists(save_dir):
os.makedirs(save_dir)
fn_rlt = osp.join(save_dir, 'ident_test_topN_details.txt')
fn_rlt2 = osp.join(save_dir, 'ident_test_topN_rlt.txt')
do_norm = not (args.no_normalize)
ftr_mat = load_mat_features(args.feature_mat_file, do_norm)
gallery_img_list, gallery_idx_list, gallery_id_list = load_image_list(
args.gallery_list_file)
gallery_ftrs = ftr_mat[np.array(gallery_idx_list)]
print "gallery_ftrs.shape: ", gallery_ftrs.shape
n_gallery = len(gallery_img_list)
if top_N > n_gallery:
top_N = n_gallery
probe_img_list, probe_idx_list, probe_id_list = load_image_list(
args.probe_list_file)
probe_ftrs = ftr_mat[np.array(probe_idx_list)]
print "probe_ftrs.shape: ", probe_ftrs.shape
n_probe = len(probe_img_list)
similarity_mat = np.dot(probe_ftrs, gallery_ftrs.T)
print "similarity_mat.shape: ", similarity_mat.shape
fp_rlt = open(fn_rlt, 'w')
top_N_cnt_ttl = np.zeros(top_N)
for i in range(n_probe):
top_N_cnt = np.zeros(top_N)
sim_row = similarity_mat[i]
top_N_idx_idx = np.argsort(-sim_row)[:top_N + 1]
top_one_idx = gallery_idx_list[top_N_idx_idx[0]]
# if i==0:
# print "probe img_name: ", probe_img_list[i]
# print "probe img_idx: ", probe_idx_list[i]
# print "probe img_id: ", probe_id_list[i]
#
# print 'sim_row.max: ', sim_row.max()
#
# print "top_1 img_name: ", gallery_img_list[top_N_idx_idx[0]]
#
# print "===>Before check top_1's img_idx\n"
# print "top_N_idx_idx:", top_N_idx_idx
# print "top_N sim: ", sim_row[top_N_idx_idx]
# print "top_N_idx:", gallery_idx_list[top_N_idx_idx]
# print "top_N_id:", gallery_id_list[top_N_idx_idx]
if top_one_idx == probe_idx_list[i]:
top_N_idx_idx = top_N_idx_idx[1:top_N + 1]
else:
top_N_idx_idx = top_N_idx_idx[0:top_N]
# if i==0:
# print "===>After check top_1's img_idx\n"
# print "top_N_idx_idx:", top_N_idx_idx
# print "top_N sim: ", sim_row[top_N_idx_idx]
# print "top_N_idx:", gallery_idx_list[top_N_idx_idx]
# print "top_N_id:", gallery_id_list[top_N_idx_idx]
for j, idx_idx in enumerate(top_N_idx_idx):
top_N_cnt[j] += (probe_id_list[i] == gallery_id_list[idx_idx])
# if i==0:
# print "top_N_cnt:", top_N_cnt
for j in range(1, top_N):
top_N_cnt[j] += top_N_cnt[j - 1]
#
# if i==0:
# print "top_N_cnt:", top_N_cnt
fp_rlt.write("%s" % probe_img_list[i])
for j in range(top_N):
fp_rlt.write("\t%5d" % top_N_cnt[j])
top_N_cnt_ttl[j] += (top_N_cnt[j] > 0)
fp_rlt.write("\n")
fp_rlt.close()
fp_rlt = open(fn_rlt2, 'w')
fp_rlt.write("args: \n{}\n\n".format(args))
fp_rlt.write("Num of gallery features: %d\n" % n_gallery)
fp_rlt.write("Num of probe features: %d\n\n" % n_probe)
top_N_ratio = top_N_cnt_ttl / n_probe
print "top_N_cnt_ttl: ", top_N_cnt_ttl
print "top_N_ratio: ", top_N_ratio
fp_rlt.write("TOP_N \tcnt \t ratio\n")
fp_rlt.write("----------------------\n")
for j in range(top_N):
fp_rlt.write("top_%d\t%5d\t%5.4f\n" %
(j + 1, top_N_cnt_ttl[j], top_N_ratio[j]))
fp_rlt.write("\n")
# for j in range(top_N):
# fp_rlt.write("%5d\t" % top_N_cnt_ttl[j])
# fp_rlt.write("\n")
#
# for j in range(top_N):
# fp_rlt.write("%5d\t" % top_N_ratio[j])
fp_rlt.write("\n")
fp_rlt.close()