def read_list(path_in): with open(path_in) as fin: identities = [] last = [-1, -1] _id = 1 while True: line = fin.readline() if not line: break item = edict() item.flag = 0 print('face', face_preprocess.parse_lst_line(line)) print('line read_list', line) item.image_path, label, item.bbox, item.landmark, item.aligned = face_preprocess.parse_lst_line( line) if not item.aligned and item.landmark is None: print('ignore line', line) continue item.id = _id item.label = [label, item.aligned] yield item if label != last[0]: if last[1] >= 0: identities.append((last[1], _id)) last[0] = label last[1] = _id _id += 1
def main(args): global image_shape global net print(args) ctx = [] cvd = os.environ['CUDA_VISIBLE_DEVICES'].strip() if len(cvd) > 0: for i in range(len(cvd.split(','))): ctx.append(mx.gpu(i)) if len(ctx) == 0: ctx = [mx.cpu()] print('use cpu') else: print('gpu num:', len(ctx)) image_shape = [int(x) for x in args.image_size.split(',')] vec = args.model.split(',') assert len(vec) > 1 prefix = vec[0] epoch = int(vec[1]) print('loading', prefix, epoch) net = edict() net.ctx = ctx net.sym, net.arg_params, net.aux_params = mx.model.load_checkpoint( prefix, epoch) # net.arg_params, net.aux_params = ch_dev(net.arg_params, net.aux_params, net.ctx) all_layers = net.sym.get_internals() net.sym = all_layers['fc1_output'] net.model = mx.mod.Module(symbol=net.sym, context=net.ctx, label_names=None) net.model.bind(data_shapes=[('data', (args.batch_size, 3, image_shape[1], image_shape[2]))]) net.model.set_params(net.arg_params, net.aux_params) i = 0 features_all = np.zeros((data_size, emb_size), dtype=np.float32) fstart = 0 buffer = [] for line in open(args.input, 'r'): if i % 1000 == 0: print("processing ", i) i += 1 image_path, label, bbox, landmark, aligned = face_preprocess.parse_lst_line( line) buffer.append((image_path, landmark)) if len(buffer) == args.batch_size: embedding = get_feature(buffer) buffer = [] fend = fstart + embedding.shape[0] # print('writing', fstart, fend) features_all[fstart:fend, :] = embedding fstart = fend if len(buffer) > 0: embedding = get_feature(buffer) fend = fstart + embedding.shape[0] print('writing', fstart, fend) features_all[fstart:fend, :] = embedding write_bin(args.output, features_all) os.system("bypy upload %s" % args.output)
def read_list(path_in): with open(path_in) as fin: identities = [] last = [-1, -1] _id = 1 while True: line = fin.readline() if not line: break item = edict() item.flag = 0 item.image_path, item.label, item.bbox, item.landmark, item.aligned = face_preprocess.parse_lst_line(line) item.id = _id yield item if item.label!=last[0]: if last[1]>=0: identities.append( (last[1], _id) ) last[0] = item.label last[1] = _id _id+=1 identities.append( (last[1], _id) ) item = edict() item.flag = 1 item.id = 0 item.label = [float(_id), float(_id+len(identities))] yield item for identity in identities: item = edict() item.flag = 2 item.id = _id _id+=1 item.label = [float(identity[0]), float(identity[1])] yield item
def read_list(path_in): idToPathFile = open("id_to_path.txt", "w") with open(path_in) as fin: identities = [] last = [-9999999, -1] _id = 1 baseDir = os.path.dirname(path_in) while True: line = fin.readline() if not line: break item = edict() item.flag = 0 item.image_path, label, item.bbox, item.landmark, item.aligned = face_preprocess.parse_lst_line( line) if not os.path.exists(item.image_path): pathItemList = item.image_path.split('/') item.image_path = os.path.join(baseDir, pathItemList[-3], pathItemList[-2], pathItemList[-1]) if not os.path.exists(item.image_path): print('path error! ignore line: ', line) continue if not item.aligned and item.landmark is None: print('ignore line: ', line) continue item.id = _id #item index item.label = [label, item.aligned] # record id<==>path idToPathFile.write("%d, %s\n" % (item.id, item.image_path)) yield item if label != last[ 0]: #save (label,id ) relation info. endding of each class if last[1] >= 0: identities.append((last[1], _id)) last[0] = label last[1] = _id _id += 1 identities.append((last[1], _id)) item = edict() item.flag = 2 item.id = 0 item.label = [float(_id), float(_id + len(identities)) ] #last _id , last _id + len(ids) yield item for identity in identities: item = edict() item.flag = 2 item.id = _id _id += 1 item.label = [float(identity[0]), float(identity[1])] yield item # each endding item of each class idToPathFile.close()
def read_list(path_in): with open(path_in) as fin: _id = 0 while True: line = fin.readline() if not line: break item = edict() item.image_path, item.label, item.bbox, item.landmark, item.aligned = face_preprocess.parse_lst_line( line) item.id = _id _id += 1 yield item
def read_list(path_in): with open(path_in) as fin: identities = [] last = [-1, -1] # range's [label, start id] _id = 1 # idx == 1 while True: line = fin.readline() if not line: break item = edict() item.flag = 0 item.image_path, label, item.bbox, item.landmark, item.aligned = face_preprocess.parse_lst_line( line) #if not item.aligned and item.landmark is None: #print('ignore line', line) #continue item.id = _id item.label = [label, item.aligned] if item.bbox is not None: item.label += list(item.bbox) if item.landmark is not None: item.label += item.landmark yield item if label != last[0]: if last[1] >= 0: identities.append( (last[1], _id)) # identity range for last label last[0] = label last[1] = _id _id += 1 identities.append((last[1], _id)) item = edict() item.flag = 2 item.id = 0 # idx == 0 item.label = [float(_id), float(_id + len(identities)) ] # image end idx, identity end idx yield item for identity in identities: # record all range item = edict() item.flag = 2 item.id = _id _id += 1 item.label = [float(identity[0]), float(identity[1])] yield item
def read_list(path_in): with open(path_in) as fin: identities = [] last = [-1, -1] _id = 1 while True: line = fin.readline() if not line: break item = edict() item.flag = 0 item.image_path, label, item.bbox, item.landmark, item.aligned = face_preprocess.parse_lst_line(line) # item.is_card = int(path.split(item.image_path)[1][0] == 'X') if not item.aligned and item.landmark is None: #print('ignore line', line) continue item.id = _id item.label = [label, item.aligned] yield item if label!=last[0]: if last[1]>=0: identities.append( (last[1], _id) ) last[0] = label last[1] = _id _id+=1 identities.append( (last[1], _id) ) item = edict() item.flag = 2 item.id = 0 item.label = [float(_id), float(_id+len(identities))] # item.is_card = 2 yield item for identity in identities: item = edict() item.flag = 2 item.id = _id _id+=1 item.label = [float(identity[0]), float(identity[1])] # item.is_card = 2 yield item
def read_list(path_in): with open(path_in) as fin: identities = [] last = [-1, -1] _id = 1 while True: line = fin.readline() if not line: break item = edict() item.flag = 0 item.image_path, label, item.bbox, item.landmark, item.aligned = face_preprocess.parse_lst_line(line) if not item.aligned and item.landmark is None: #print('ignore line', line) continue item.id = _id item.label = [label, item.aligned] yield item if label!=last[0]: if last[1]>=0: identities.append( (last[1], _id) ) last[0] = label last[1] = _id _id+=1 identities.append( (last[1], _id) ) item = edict() item.flag = 2 item.id = 0 item.label = [float(_id), float(_id+len(identities))] yield item for identity in identities: item = edict() item.flag = 2 item.id = _id _id+=1 item.label = [float(identity[0]), float(identity[1])] yield item
def main(args): out_algo = args.suffix if len(args.algo) > 0: out_algo = args.algo fs_noise_map = {} for line in open(args.facescrub_noises, 'r'): if line.startswith('#'): continue line = line.strip() fname = line.split('.')[0] p = fname.rfind('_') fname = fname[0:p] fs_noise_map[line] = fname print(len(fs_noise_map)) i = 0 fname2center = {} noises = [] for line in open(args.facescrub_lst, 'r'): if i % 1000 == 0: print("reading fs", i) i += 1 image_path, label, bbox, landmark, aligned = face_preprocess.parse_lst_line( line) assert aligned == True _path = image_path.split('/') a, b = _path[-2], _path[-1] feature_path = os.path.join(args.facescrub_feature_dir, a, "%s_%s.bin" % (b, args.suffix)) feature_dir_out = os.path.join(args.facescrub_feature_dir_out, a) if not os.path.exists(feature_dir_out): os.makedirs(feature_dir_out) feature_path_out = os.path.join(args.facescrub_feature_dir_out, a, "%s_%s.bin" % (b, out_algo)) #print(b) if not b in fs_noise_map: #shutil.copyfile(feature_path, feature_path_out) feature = load_bin(feature_path) write_bin(feature_path_out, feature) if not a in fname2center: fname2center[a] = np.zeros((feature_dim + feature_ext, ), dtype=np.float32) fname2center[a] += feature else: #print('n', b) noises.append((a, b)) print(len(noises)) for k in noises: a, b = k assert a in fname2center center = fname2center[a] g = np.zeros((feature_dim + feature_ext, ), dtype=np.float32) g2 = np.random.uniform(-0.001, 0.001, (feature_dim, )) g[0:feature_dim] = g2 f = center + g _norm = np.linalg.norm(f) f /= _norm feature_path_out = os.path.join(args.facescrub_feature_dir_out, a, "%s_%s.bin" % (b, out_algo)) write_bin(feature_path_out, f) mf_noise_map = {} for line in open(args.megaface_noises, 'r'): if line.startswith('#'): continue line = line.strip() _vec = line.split("\t") if len(_vec) > 1: line = _vec[1] mf_noise_map[line] = 1 print(len(mf_noise_map)) i = 0 nrof_noises = 0 for line in open(args.megaface_lst, 'r'): if i % 1000 == 0: print("reading mf", i) i += 1 image_path, label, bbox, landmark, aligned = face_preprocess.parse_lst_line( line) assert aligned == True _path = image_path.split('/') a1, a2, b = _path[-3], _path[-2], _path[-1] feature_path = os.path.join(args.megaface_feature_dir, a1, a2, "%s_%s.bin" % (b, args.suffix)) feature_dir_out = os.path.join(args.megaface_feature_dir_out, a1, a2) if not os.path.exists(feature_dir_out): os.makedirs(feature_dir_out) feature_path_out = os.path.join(args.megaface_feature_dir_out, a1, a2, "%s_%s.bin" % (b, out_algo)) bb = '/'.join([a1, a2, b]) #print(b) if not bb in mf_noise_map: feature = load_bin(feature_path) write_bin(feature_path_out, feature) #shutil.copyfile(feature_path, feature_path_out) else: feature = load_bin(feature_path, 100.0) write_bin(feature_path_out, feature) #g = np.random.uniform(-0.001, 0.001, (feature_dim,)) #print('n', bb) #write_bin(feature_path_out, g) nrof_noises += 1 print(nrof_noises)
def main(args): print(args) gpuid = args.gpu ctx = mx.gpu(gpuid) nets = [] image_shape = [int(x) for x in args.image_size.split(',')] for model in args.model.split('|'): vec = model.split(',') assert len(vec) > 1 prefix = vec[0] epoch = int(vec[1]) print('loading', prefix, epoch) net = edict() net.ctx = ctx net.sym, net.arg_params, net.aux_params = mx.model.load_checkpoint(prefix, epoch) # net.arg_params, net.aux_params = ch_dev(net.arg_params, net.aux_params, net.ctx) all_layers = net.sym.get_internals() net.sym = all_layers['fc1_output'] net.model = mx.mod.Module(symbol=net.sym, context=net.ctx, label_names=None) net.model.bind(data_shapes=[('data', (1, 3, image_shape[1], image_shape[2]))]) net.model.set_params(net.arg_params, net.aux_params) # _pp = prefix.rfind('p')+1 # _pp = prefix[_pp:] # net.patch = [int(x) for x in _pp.split('_')] # assert len(net.patch)==5 # print('patch', net.patch) nets.append(net) # megaface_lst = "/raid5data/dplearn/faceinsight_align_megaface.lst" megaface_lst = "/raid5data/dplearn/megaface/megaface_mtcnn_112x112/lst" # facescrub_lst = "/raid5data/dplearn/faceinsight_align_facescrub.lst" facescrub_lst = "/raid5data/dplearn/megaface/facescrubr/small_lst" if args.fsall > 0: facescrub_lst = "/raid5data/dplearn/megaface/facescrubr/lst" if args.skip == 0: i = 0 succ = 0 for line in open(facescrub_lst, 'r'): if i % 1000 == 0: print("writing fs", i, succ) i += 1 image_path, label, bbox, landmark, aligned = face_preprocess.parse_lst_line(line) _path = image_path.split('/') a, b = _path[-2], _path[-1] # a = a.replace(' ', '_') # b = b.replace(' ', '_') out_dir = os.path.join(facescrub_out, a) if not os.path.exists(out_dir): os.makedirs(out_dir) # file, ext = os.path.splitext(b) # image_id = int(file.split('_')[-1]) # if image_id==40499 or image_id==10788 or image_id==2367: # b = file # if len(ext)==0: # print(image_path) # image_path = image_path+".jpg" # if facescrub_aligned_root is not None: # _vec = image_path.split('/') # _image_path = os.path.join(facescrub_aligned_root, _vec[-2], _vec[-1]) # _base, _ext = os.path.splitext(_image_path) # if _ext=='.gif': # _image_path = _base+".jpg" # print('changing', _image_path) # if os.path.exists(_image_path): # image_path = _image_path # bbox = None # landmark = None # aligned = True # else: # print("not aligned:",_image_path) feature = get_feature(image_path, bbox, landmark, nets, image_shape, True, aligned, args.mean) if feature is None: print('feature none', image_path) continue # print(np.linalg.norm(feature)) out_path = os.path.join(out_dir, b + "_%s_%dx%d.bin" % (args.algo, image_shape[1], image_shape[2])) write_bin(out_path, feature) succ += 1 print('fs stat', i, succ) # return if args.mf == 0: return i = 0 succ = 0 for line in open(megaface_lst, 'r'): if i % 1000 == 0: print("writing mf", i, succ) i += 1 if i <= args.skip: continue image_path, label, bbox, landmark, aligned = face_preprocess.parse_lst_line(line) assert aligned == True _path = image_path.split('/') a1, a2, b = _path[-3], _path[-2], _path[-1] out_dir = os.path.join(megaface_out, a1, a2) if not os.path.exists(out_dir): os.makedirs(out_dir) # continue # print(landmark) feature = get_feature(image_path, bbox, landmark, nets, image_shape, True, aligned, args.mean) if feature is None: continue out_path = os.path.join(out_dir, b + "_%s_%dx%d.bin" % (args.algo, image_shape[1], image_shape[2])) # print(out_path) write_bin(out_path, feature) succ += 1 print('mf stat', i, succ)
def main(args): out_algo = args.suffix if len(args.algo)>0: out_algo = args.algo fs_noise_map = {} for line in open(args.facescrub_noises, 'r'): if line.startswith('#'): continue line = line.strip() fname = line.split('.')[0] p = fname.rfind('_') fname = fname[0:p] fs_noise_map[line] = fname print(len(fs_noise_map)) i=0 fname2center = {} noises = [] for line in open(args.facescrub_lst, 'r'): if i%1000==0: print("reading fs",i) i+=1 image_path, label, bbox, landmark, aligned = face_preprocess.parse_lst_line(line) assert aligned==True _path = image_path.split('/') a, b = _path[-2], _path[-1] feature_path = os.path.join(args.facescrub_feature_dir, a, "%s_%s.bin"%(b, args.suffix)) feature_dir_out = os.path.join(args.facescrub_feature_dir_out, a) if not os.path.exists(feature_dir_out): os.makedirs(feature_dir_out) feature_path_out = os.path.join(args.facescrub_feature_dir_out, a, "%s_%s.bin"%(b, out_algo)) #print(b) if not b in fs_noise_map: #shutil.copyfile(feature_path, feature_path_out) feature = load_bin(feature_path) write_bin(feature_path_out, feature) if not a in fname2center: fname2center[a] = np.zeros((feature_dim+feature_ext,), dtype=np.float32) fname2center[a] += feature else: #print('n', b) noises.append( (a,b) ) print(len(noises)) for k in noises: a,b = k assert a in fname2center center = fname2center[a] g = np.zeros( (feature_dim+feature_ext,), dtype=np.float32) g2 = np.random.uniform(-0.001, 0.001, (feature_dim,)) g[0:feature_dim] = g2 f = center+g _norm=np.linalg.norm(f) f /= _norm feature_path_out = os.path.join(args.facescrub_feature_dir_out, a, "%s_%s.bin"%(b, out_algo)) write_bin(feature_path_out, f) mf_noise_map = {} for line in open(args.megaface_noises, 'r'): if line.startswith('#'): continue line = line.strip() _vec = line.split("\t") if len(_vec)>1: line = _vec[1] mf_noise_map[line] = 1 print(len(mf_noise_map)) i=0 nrof_noises = 0 for line in open(args.megaface_lst, 'r'): if i%1000==0: print("reading mf",i) i+=1 image_path, label, bbox, landmark, aligned = face_preprocess.parse_lst_line(line) assert aligned==True _path = image_path.split('/') a1, a2, b = _path[-3], _path[-2], _path[-1] feature_path = os.path.join(args.megaface_feature_dir, a1, a2, "%s_%s.bin"%(b, args.suffix)) feature_dir_out = os.path.join(args.megaface_feature_dir_out, a1, a2) if not os.path.exists(feature_dir_out): os.makedirs(feature_dir_out) feature_path_out = os.path.join(args.megaface_feature_dir_out, a1, a2, "%s_%s.bin"%(b, out_algo)) bb = '/'.join([a1, a2, b]) #print(b) if not bb in mf_noise_map: feature = load_bin(feature_path) write_bin(feature_path_out, feature) #shutil.copyfile(feature_path, feature_path_out) else: feature = load_bin(feature_path, 100.0) write_bin(feature_path_out, feature) #g = np.random.uniform(-0.001, 0.001, (feature_dim,)) #print('n', bb) #write_bin(feature_path_out, g) nrof_noises+=1 print(nrof_noises)
def main(args): print(args) gpuid = args.gpu ctx = mx.gpu(gpuid) nets = [] image_shape = [int(x) for x in args.image_size.split(',')] for model in args.model.split('|'): vec = model.split(',') assert len(vec)>1 prefix = vec[0] epoch = int(vec[1]) print('loading',prefix, epoch) net = edict() net.ctx = ctx net.sym, net.arg_params, net.aux_params = mx.model.load_checkpoint(prefix, epoch) #net.arg_params, net.aux_params = ch_dev(net.arg_params, net.aux_params, net.ctx) all_layers = net.sym.get_internals() net.sym = all_layers['fc1_output'] net.model = mx.mod.Module(symbol=net.sym, context=net.ctx, label_names = None) net.model.bind(data_shapes=[('data', (1, 3, image_shape[1], image_shape[2]))]) net.model.set_params(net.arg_params, net.aux_params) #_pp = prefix.rfind('p')+1 #_pp = prefix[_pp:] #net.patch = [int(x) for x in _pp.split('_')] #assert len(net.patch)==5 #print('patch', net.patch) nets.append(net) #megaface_lst = "/raid5data/dplearn/faceinsight_align_megaface.lst" megaface_lst = "/raid5data/dplearn/megaface/megaface_mtcnn_112x112/lst" #facescrub_lst = "/raid5data/dplearn/faceinsight_align_facescrub.lst" facescrub_lst = "/raid5data/dplearn/megaface/facescrubr/small_lst" if args.fsall>0: facescrub_lst = "/raid5data/dplearn/megaface/facescrubr/lst" if args.skip==0: i = 0 succ = 0 for line in open(facescrub_lst, 'r'): if i%1000==0: print("writing fs",i, succ) i+=1 image_path, label, bbox, landmark, aligned = face_preprocess.parse_lst_line(line) _path = image_path.split('/') a,b = _path[-2], _path[-1] #a = a.replace(' ', '_') #b = b.replace(' ', '_') out_dir = os.path.join(facescrub_out, a) if not os.path.exists(out_dir): os.makedirs(out_dir) #file, ext = os.path.splitext(b) #image_id = int(file.split('_')[-1]) #if image_id==40499 or image_id==10788 or image_id==2367: # b = file #if len(ext)==0: # print(image_path) # image_path = image_path+".jpg" #if facescrub_aligned_root is not None: # _vec = image_path.split('/') # _image_path = os.path.join(facescrub_aligned_root, _vec[-2], _vec[-1]) # _base, _ext = os.path.splitext(_image_path) # if _ext=='.gif': # _image_path = _base+".jpg" # print('changing', _image_path) # if os.path.exists(_image_path): # image_path = _image_path # bbox = None # landmark = None # aligned = True # else: # print("not aligned:",_image_path) feature = get_feature(image_path, bbox, landmark, nets, image_shape, True, aligned, args.mean) if feature is None: print('feature none', image_path) continue #print(np.linalg.norm(feature)) out_path = os.path.join(out_dir, b+"_%s_%dx%d.bin"%(args.algo, image_shape[1], image_shape[2])) write_bin(out_path, feature) succ+=1 print('fs stat',i, succ) #return if args.mf==0: return i = 0 succ = 0 for line in open(megaface_lst, 'r'): if i%1000==0: print("writing mf",i, succ) i+=1 if i<=args.skip: continue image_path, label, bbox, landmark, aligned = face_preprocess.parse_lst_line(line) assert aligned==True _path = image_path.split('/') a1, a2, b = _path[-3], _path[-2], _path[-1] out_dir = os.path.join(megaface_out, a1, a2) if not os.path.exists(out_dir): os.makedirs(out_dir) #continue #print(landmark) feature = get_feature(image_path, bbox, landmark, nets, image_shape, True, aligned, args.mean) if feature is None: continue out_path = os.path.join(out_dir, b+"_%s_%dx%d.bin"%(args.algo, image_shape[1], image_shape[2])) #print(out_path) write_bin(out_path, feature) succ+=1 print('mf stat',i, succ)