def concatN_from_list(dirList, pathList, saveDir): sourceDir = dirList[0] N = len(dirList) errorCnt = 0 for path in pathList: sourcePath = path tmpVec = np.transpose(matio.load_mat(sourcePath))[:, 0:-1] _concatedPath = path.replace(sourceDir, '') if _concatedPath.startswith('/'): _concatedPath = _concatedPath[1:] concatedPath = os.path.join(saveDir, _concatedPath) # try: for followDir in dirList[1:]: followPath = sourcePath.replace(sourceDir, followDir) untransposed = matio.load_mat(followPath) followVec = np.transpose(untransposed)[:, 0:-1] tmpVec = np.concatenate((tmpVec, followVec), axis=1) #pad the last label tmpVec = tmpVec / np.sqrt(N) tmpVec = np.concatenate( (tmpVec, np.expand_dims(np.transpose(untransposed)[:, -1], axis=0)), axis=1) createPath_write_bin(concatedPath, tmpVec.T) return
def compare(): # verify the keys ACCESS_KEY = request.form.get('ACCESS_KEY').encode("utf-8") TIMESTAMP = request.form.get('TIMESTAMP').encode("utf-8") SIGN_KEY = request.form.get('SIGN_KEY').encode("utf-8") user_id = request.form.get('user_id').encode("utf-8") id_0 = request.form.get('id_0').encode("utf-8") id_1 = request.form.get('id_1').encode("utf-8") if ACCESS_KEY and TIMESTAMP and SIGN_KEY and user_id >= 0 and id_0 >= 0 and id_1 >= 0: check_result, message = _check_security(ACCESS_KEY, TIMESTAMP, SIGN_KEY, user_id) if not check_result: return message else: return error_resp(2, "LACK_PARAM ERROR") file_path1 = os.path.join(app.config['FEATURE_FOLDER'], "{}.bin".format(id_0)) file_path2 = os.path.join(app.config['FEATURE_FOLDER'], "{}.bin".format(id_1)) if (not os.path.exists(file_path1) or not os.path.exists(file_path2)): return error_resp(2, "FACE_ID ERROR") feat1 = load_mat(file_path1) feat2 = load_mat(file_path2) score = eng.calc_score_result(feat1, feat2) resp = jsonify(status=1, result="Same Person" if score > 0.5 else "Different Person", similarity="%.3f" % score) resp.headers['Access-Control-Allow-Origin'] = '*' return resp
def clean_intra_noise(root_folder, feature_path, fea_dims, threshold): print('Start load feature') label_featurelist_dict = get_label_featurelist_dict(feature_path) print('Finished load feature') lable_center_dict = dict() # Calulate feature center for key in label_featurelist_dict: feature_sum = np.zeros(fea_dims, dtype=np.float64) cur_fea_list = label_featurelist_dict[key] for i in range(len(cur_fea_list)): full_path = os.path.join(root_folder, cur_fea_list[i]) x_vec = matio.load_mat(full_path).flatten() feature_sum += x_vec feature_center = feature_sum / len(cur_fea_list) lable_center_dict[key] = feature_center # compare insta distance noise_img = [] for key in label_featurelist_dict: cur_fea_list = label_featurelist_dict[key] for i in range(len(cur_fea_list)): full_path = os.path.join(root_folder, cur_fea_list[i]) x_vec = matio.load_mat(full_path).flatten() center = lable_center_dict[key] sim = np.dot(x_vec, center) / (np.linalg.norm(x_vec, ord=2) * np.linalg.norm(center, ord=2)) if sim < threshold: #print(cur_fea_list[i] + ' ' + str(dist)) noise_img.append(cur_fea_list[i] + ' ' + str(sim)) return noise_img
def main(args): print('===> args:\n', args) image_list = args.image_list feature_dir_1 = args.feature_dir_1 feature_dir_2 = args.feature_dir_2 save_type = args.save_format feature_len = args.feature_dims i = 0 with open(image_list, 'r') as f: lines = f.readlines() print('###### read features nums: %d ######' % (len(lines))) X = np.zeros(shape=(len(lines), feature_len)) for line in lines: feature_name = line.strip() + save_type feature_path_1 = os.path.join(feature_dir_1, feature_name) x_vec_1 = np.transpose(matio.load_mat(feature_path_1)) feature_path_2 = os.path.join(feature_dir_2, feature_name) x_vec_2 = np.transpose(matio.load_mat(feature_path_2)) x_vec = np.concatenate((x_vec_1, x_vec_2), axis=0) X[i] = x_vec i = i + 1 print('###### success load feature nums: %d ######' % i) print(X.shape) #ipca '''
def concatN_from_list(dirList, pathList, saveDir): sourceDir = dirList[0] errorCnt = 0 for path in pathList: sourcePath = path tmpVec = np.transpose(matio.load_mat(sourcePath)) _concatedPath = path.replace(sourceDir, '') if _concatedPath.startswith('/'): _concatedPath = _concatedPath[1:] concatedPath = os.path.join(saveDir, _concatedPath) # try: for followDir in dirList[1:]: followPath = sourcePath.replace(sourceDir, followDir) untransposed = matio.load_mat(followPath) followVec = np.transpose(untransposed) tmpVec = np.concatenate((tmpVec, followVec), axis=1) #pad the last label #tmpVec = np.concatenate((tmpVec, np.expand_dims(followVec[:, -1], axis=0)), axis=1) createPath_write_bin(concatedPath, tmpVec.T) # except Exception e: # print e.message() # errorCnt += 1 # print errorCnt, "errors" return
def clean_intra_noise(root_folder, feature_path, fea_dims, threshold): print('Start load feature') label_featurelist_dict = get_label_featurelist_dict(feature_path) print('Finished load feature') lable_center_dict = dict() # Calulate feature center for key in label_featurelist_dict: feature_sum = np.zeros(fea_dims, dtype=np.float64) cur_fea_list = label_featurelist_dict[key] feature_center = np.zeros(fea_dims, dtype=np.float64) fea_num = len(cur_fea_list) is_finised_flag = False for i in range(fea_num): count = 0 full_path = os.path.join(root_folder, cur_fea_list[i]) x_vec_1 = matio.load_mat(full_path).flatten() for j in range(fea_num): full_path2 = os.path.join(root_folder, cur_fea_list[j]) x_vec_2 = matio.load_mat(full_path2).flatten() sim = np.dot(x_vec_1, x_vec_2) / ( np.linalg.norm(x_vec_1, ord=2) * np.linalg.norm(x_vec_2, ord=2) + +0.0000001) if sim > 0.6: feature_sum = feature_sum + x_vec_2 count = count + 1 if count > (fea_num * 0.5): is_finised_flag = True break if is_finised_flag == True: feature_center = feature_sum / count break lable_center_dict[key] = feature_center # compare insta distance noise_img = [] for key in label_featurelist_dict: cur_fea_list = label_featurelist_dict[key] for i in range(len(cur_fea_list)): full_path = os.path.join(root_folder, cur_fea_list[i]) x_vec = matio.load_mat(full_path).flatten() center = lable_center_dict[key] sim = np.dot(x_vec, center) / ( np.linalg.norm(x_vec, ord=2) * np.linalg.norm(center, ord=2) + 0.0000001) if sim < threshold: print(cur_fea_list[i] + ' ' + str(sim)) noise_img.append(cur_fea_list[i] + ' ' + str(sim)) return noise_img
def recognize(): # verify the keys ACCESS_KEY = request.form.get('ACCESS_KEY').encode("utf-8") TIMESTAMP = request.form.get('TIMESTAMP').encode("utf-8") SIGN_KEY = request.form.get('SIGN_KEY').encode("utf-8") user_id = request.form.get('user_id').encode("utf-8") id = request.form.get('id').encode("utf-8") if ACCESS_KEY and TIMESTAMP and SIGN_KEY and user_id >= 0 and id >= 0: check_result, message = _check_security(ACCESS_KEY, TIMESTAMP, SIGN_KEY, user_id) if not check_result: return message else: return error_resp(2, "LACK_PARAM ERROR") file_path = os.path.join(app.config['FEATURE_FOLDER'], "{}.bin".format(id)) if (not os.path.exists(file_path)): return error_resp(2, "FACE_ID ERROR") feat = load_mat(file_path) names, scores = inner_recognize(feat) min_value = min(scores) scores = [s - min_value for s in scores] sum_value = sum(scores) norm_scores = ["%.2f" % np.divide(s, sum_value) for s in scores] results = [] for i in xrange(len(norm_scores)): results.append(serialize3(names[i], norm_scores[i])) resp = jsonify(status=1, results=results) resp.headers['Access-Control-Allow-Origin'] = '*' return resp
def concat2_from_list(aDir, bDir, pathList, saveDir): for path in pathList: aPath = path bPath = path.replace(aDir, bDir) _concatedPath = path.replace(aDir, '') concatedPath = os.path.join(saveDir, _concatedPath) try: aFeatureVec = np.transpose(matio.load_mat(aPath)) bFeatureVec = np.transpose(matio.load_mat(bPath)) concatedFeatureVec = np.concatenate((aFeatureVec, bFeatureVec), axis=1) #print concatedPath createPath_write_bin(concatedPath, concatedFeatureVec) except: print 'unable to process' return
def main(args): print('===> args:\n', args) image_list = args.image_list feature_dir = args.feature_dir save_type = args.save_format feature_len = args.feature_dims i = 0 with open(image_list, 'r') as f: lines = f.readlines() print('###### read features nums: %d ######' % (len(lines))) X = np.zeros(shape=(len(lines), feature_len)) for line in lines: feature_name = line.strip() + save_type feature_path = os.path.join(feature_dir, feature_name) x_vec = np.transpose(matio.load_mat(feature_path)) X[i] = x_vec i = i + 1 print('###### success load feature nums: %d ######' % i) print(X.shape) #ipca ipca = IncrementalPCA(n_components=args.n_components) ipca.fit(X) print('###### PCA Done! ######') joblib.dump(ipca, args.ipca_save_path) print('components num: %d' % ipca.n_components) sum_variance_ratio = 0 for i in range(ipca.n_components): sum_variance_ratio += ipca.explained_variance_ratio_[i] print('sum_variance_ratio: %f' % sum_variance_ratio)
def read_feature(fea_root, path): fea_dict = dict() with open(path) as f: lines = f.readlines() for line in lines: full_path = os.path.join(fea_root, line.strip()) x_vec = matio.load_mat(full_path).flatten() fea_dict[line.strip()] = x_vec return fea_dict
def load_cele_features(celefeature_folder): features = [] names = [] for the_file in os.listdir(celefeature_folder): name = the_file.rsplit('#', 1)[0] file_path = os.path.join(celefeature_folder, the_file) feature = load_mat(file_path) features.append(feature) names.append(name) return features, names
def load_feature_list(rel_list, feature_dir, suffix): total_size = len(rel_list) featlist = [] for i in range(total_size): # make fullpath feat_path = os.path.join(feature_dir, rel_list[i]) feat_path = feat_path + suffix feat = load_mat(feat_path) featlist.append(feat) if i % 100 == 0: print('\t%d/%d\t%s' % (i, total_size, rel_list[i])) return featlist
def load_feat(feat_file, flatten=True): feat = None if feat_file.endswith('npy'): feat = np.load(feat_file) elif feat_file.endswith('bin'): feat = load_mat(feat_file) else: raise Exception( 'Unsupported feature file. Only support .npy and .bin (OpenCV Mat file)' ) if flatten: feat = feat.flatten() return feat
def read_feature(root_folder, feature_path): label_feature_dict = get_img_list(feature_path) feature_matrix = [] for key in label_feature_dict: each_class_feature = [] cur_fea_list = label_feature_dict[key] for i in range(len(cur_fea_list)): full_path = os.path.join(root_folder, cur_fea_list[i]) x_vec = matio.load_mat(full_path).flatten() each_class_feature.append(x_vec) feature_matrix.append(np.array(each_class_feature, dtype=np.float64)) return feature_matrix
def main(args): root_folder = args.feature_dir img_list = args.image_list save_list = args.out_result_list threshold = args.threshold with open(img_list) as f, open(save_list, 'w') as f2: lines = f.readlines() print('All Feature Num: %d' % len(lines)) for line in lines: fea_name = line.strip() + fea_type if not os.path.exists(osp.join(root_folder, fea_name)): continue fea = matio.load_mat(osp.join(root_folder, fea_name)).flatten() norm = np.linalg.norm(fea) if norm < threshold: f2.write(line.strip() + ' ' + str(norm) + '\n')
def clean_inter_noise(root_folder, feature_path, fea_dims,threshold): label_featurelist_dict = get_label_featurelist_dict(feature_path) print('Finished load feature') lable_center_dict = dict() # Calulate feature center for key in label_featurelist_dict: feature_sum = np.zeros(fea_dims, dtype=np.float64) cur_fea_list = label_featurelist_dict[key] for i in range(len(cur_fea_list)): full_path = os.path.join(root_folder, cur_fea_list[i]) x_vec = matio.load_mat(full_path).flatten() feature_sum += x_vec feature_center = feature_sum / len(cur_fea_list) lable_center_dict[key] = feature_center # compare insta distance print('Finished Cal class center') label_visited_dict = dict() for key in lable_center_dict: label_visited_dict[key] = 0 print('Start merge same class') #same_class = [] out_result = [] for key in lable_center_dict: tmp_class = [] if label_visited_dict[key] == 0: tmp_class.append(key) label_visited_dict[key] = 1 cur_fea_center = lable_center_dict[key] for key2 in lable_center_dict: if key != key2 and label_visited_dict[key2] == 0: #label_visited_dict[key2] = 1 com_fea_center = lable_center_dict[key2] dist = np.dot(cur_fea_center, com_fea_center) / (np.linalg.norm(cur_fea_center, ord=2) * np.linalg.norm(com_fea_center, ord=2)) #print(str(key)+' ' +str(key2) + ' ' + str(dist)) if dist > threshold: print(str(key)+' ' +str(key2) + ' ' + str(dist)) label_visited_dict[key2] = 1 tmp_class.append(key2) out_result.append(str(key)+' ' +str(key2) + ' ' + str(dist)) return out_result
def test_set(distractor_path, probe, feature_dir, suffix): # load distractor with open(distractor_path, 'r') as f: data = json.load(f) distractor_list = data['path'] # probe set T = len(probe['id']) probe_set = set() for id in probe['id']: probe_set.add(id) ids = list(probe_set) N = len(ids) pclass = {} for id in ids: pclass[id] = [] for i in range(len(probe['id'])): id = probe['id'][i] pclass[id].append(i) probe_id_list = probe['id'] probe_feat_list = probe['feat'] correct = 0 wrong = 0 # class 1,..., N for id in pclass: # i,...,M S = pclass[id] M = len(S) print('test id:%s' % id) for i in range(M): min_dist = 100000 min_idx = 0 probe_idx = S[i] probe_feat = probe_feat_list[probe_idx] probe_id = probe_id_list[probe_idx] # match distractors for j in range(len(distractor_list)): # make fullpath feat_path = os.path.join(feature_dir, distractor_list[j]) feat_path = feat_path + suffix feat = load_mat(feat_path) dist = L2_distance(feat, probe_feat) if dist < min_dist: min_dist = dist min_idx = j p2_dist = 100000 p2_idx = 0 # match probe ''' for j in range(T): if j == probe_idx: continue dist = L2_distance(probe_feat_list[j],probe_feat) if dist < p2_dist: p2_dist = dist p2_idx = j ''' for c in ids: if c != id: SS = pclass[c] MM = len(SS) II = np.random.randint(MM) j = SS[II] else: II = np.random.randint(M - 1) II = II if II < i else (II + 1) j = S[II] dist = L2_distance(probe_feat_list[j], probe_feat) if dist < p2_dist: p2_dist = dist p2_idx = j if p2_dist < min_dist: if probe['id'][p2_idx] == probe_id: print("TTT:%s\t %s %f" % (probe['path'][probe_idx], probe['path'][p2_idx], p2_dist)) else: print("FFF:%s\t %s %f" % (probe['path'][probe_idx], probe['path'][p2_idx], p2_dist)) else: print("FFF:%s\t %s %f" % (probe['path'][probe_idx], distractor_list[min_idx], min_dist)) # compare if p2_dist < min_dist and probe_id_list[p2_idx] == probe_id: correct += 1 else: wrong += 1 rank1_ratio = float(correct) / (correct + wrong) print('RandK-1:%f %d/%d' % (rank1_ratio, correct, (correct + wrong)))
save_type = '_feat.bin' save_dir = '/workspace/data/face-idcard-1M/features/idcard1M-features-insightface-r100-spa-m2.0-ep96/pca-320-split0' if not osp.exists(save_dir): os.makedirs(save_dir) i = 0 with open(feature_list_path, 'r') as f: lines = f.readlines() print('###### read features nums: %d ######' % (len(lines))) for line in lines: sub_dir = line.split('/')[0] final_dir = os.path.join(save_dir, sub_dir) if not osp.exists(final_dir): os.makedirs(final_dir) feature_path = osp.join(feature_dir, line.strip() + save_type) new_feature_path = osp.join( final_dir, line.strip().split('/')[-1] + save_type) #print new_feature_path x_vec = np.transpose(matio.load_mat(feature_path)) new_x_vec = ipca.transform(x_vec) matio.save_mat(new_feature_path, new_x_vec.T) i = i + 1 if i % 100000 == 0: print('####### Save index %d feature ######' % i) print('###### Finished Feature Reduce Dims, %d ######' % i)
import matio import argparse import os import sys import os.path as osp fea_type = '_feat.bin' def AugDistance(fea1, fea2): pass if __name__ == '__main__': #root_folder = '/workspace/data/vgg-features_0410/vggface-r100-spa-m2.0-ep96/vggface2_train_aligned_112x112/aligned_imgs' root_folder = '/workspace/data/face-idcard-1M/features/insightface-r100-spa-m2.0-ep96/' with open('../../data/pair.txt') as f, open('./result.txt', 'w') as f2: lines = f.readlines() for line in lines: fea1_name = line.strip().split(' ')[0] + fea_type fea2_name = line.strip().split(' ')[1] + fea_type fea1 = matio.load_mat(osp.join(root_folder, fea1_name)).flatten() fea2 = matio.load_mat(osp.join(root_folder, fea2_name)).flatten() dist = np.dot(fea1, fea2) / (np.linalg.norm(fea1, ord=2) * np.linalg.norm(fea2, ord=2)) f2.write('%s %s: %f\n' % (fea1_name, fea2_name, dist))
def main(args): print('===> args:\n', args) image_list = args.image_list feature_dir = args.feature_dir ipca_model_path = args.ipca_model_path save_format = args.save_format out_dir = args.out_dir if not osp.exists(out_dir): os.makedirs(out_dir) #load pca model ipca = joblib.load(ipca_model_path) print('components num: %d' % ipca.n_components) print('explained_variance_ratio: %s' % str(ipca.explained_variance_ratio_.shape)) sum_variance_ratio = 0 for i in range(ipca.n_components): sum_variance_ratio += ipca.explained_variance_ratio_[i] print('sum_variance_ratio: %f' % sum_variance_ratio) succ_num = 0 failed_num = 0 with open(image_list, 'r') as f: lines = f.readlines() print('###### read features nums: %d ######' % (len(lines))) for line in lines: sub_dir_list = line.split('/') sub_dir = '' for i in range(len(sub_dir_list) - 1): sub_dir = osp.join(sub_dir, sub_dir_list[i]) final_dir = os.path.join(out_dir, sub_dir) if not osp.exists(final_dir): os.makedirs(final_dir) feature_name = line.strip() + save_format feature_path = osp.join(feature_dir, feature_name) new_feature_path = osp.join( final_dir, line.strip().split('/')[-1] + save_format) if not osp.exists(feature_path): print('Not existed: %s' % feature_path) failed_num = failed_num + 1 else: x_vec = np.transpose(matio.load_mat(feature_path)) #print('pca after') x_vec_by_pca = ipca.transform(x_vec) feature_len = x_vec_by_pca.shape[1] norm_x_vec = np.zeros(shape=(1, feature_len)) norm_x_vec = 1.0 * x_vec_by_pca / ( np.linalg.norm(x_vec_by_pca, ord=2) + 0.000001) #feature is NAN if np.isnan(norm_x_vec).sum() > 0: print('%s is Nan' % feature_path) else: #matio.save_mat(new_feature_path, norm_x_vec.T) write_bin(new_feature_path, norm_x_vec.T) succ_num = succ_num + 1 if succ_num % 10000 == 0: print('####### Save index %d feature ######' % succ_num) print('###### Finished Feature Reduce Dims, %d ######' % succ_num) print('###### Failed Feature Reduce Dims, %d ######' % failed_num)