def read_list(path_in):
with open(path_in) as fin:
identities = []
last = [-1, -1]
_id = 1
while True:
line = fin.readline()
if not line:
break
item = edict()
item.flag = 0
print('face', face_preprocess.parse_lst_line(line))
print('line read_list', line)
item.image_path, label, item.bbox, item.landmark, item.aligned = face_preprocess.parse_lst_line(
line)
if not item.aligned and item.landmark is None:
print('ignore line', line)
continue
item.id = _id
item.label = [label, item.aligned]
yield item
if label != last[0]:
if last[1] >= 0:
identities.append((last[1], _id))
last[0] = label
last[1] = _id
_id += 1
def main(args):
global image_shape
global net
print(args)
ctx = []
cvd = os.environ['CUDA_VISIBLE_DEVICES'].strip()
if len(cvd) > 0:
for i in range(len(cvd.split(','))):
ctx.append(mx.gpu(i))
if len(ctx) == 0:
ctx = [mx.cpu()]
print('use cpu')
else:
print('gpu num:', len(ctx))
image_shape = [int(x) for x in args.image_size.split(',')]
vec = args.model.split(',')
assert len(vec) > 1
prefix = vec[0]
epoch = int(vec[1])
print('loading', prefix, epoch)
net = edict()
net.ctx = ctx
net.sym, net.arg_params, net.aux_params = mx.model.load_checkpoint(
prefix, epoch)
# net.arg_params, net.aux_params = ch_dev(net.arg_params, net.aux_params, net.ctx)
all_layers = net.sym.get_internals()
net.sym = all_layers['fc1_output']
net.model = mx.mod.Module(symbol=net.sym,
context=net.ctx,
label_names=None)
net.model.bind(data_shapes=[('data', (args.batch_size, 3, image_shape[1],
image_shape[2]))])
net.model.set_params(net.arg_params, net.aux_params)
i = 0
features_all = np.zeros((data_size, emb_size), dtype=np.float32)
fstart = 0
buffer = []
for line in open(args.input, 'r'):
if i % 1000 == 0:
print("processing ", i)
i += 1
image_path, label, bbox, landmark, aligned = face_preprocess.parse_lst_line(
line)
buffer.append((image_path, landmark))
if len(buffer) == args.batch_size:
embedding = get_feature(buffer)
buffer = []
fend = fstart + embedding.shape[0]
# print('writing', fstart, fend)
features_all[fstart:fend, :] = embedding
fstart = fend
if len(buffer) > 0:
embedding = get_feature(buffer)
fend = fstart + embedding.shape[0]
print('writing', fstart, fend)
features_all[fstart:fend, :] = embedding
write_bin(args.output, features_all)
os.system("bypy upload %s" % args.output)
def read_list(path_in):
with open(path_in) as fin:
identities = []
last = [-1, -1]
_id = 1
while True:
line = fin.readline()
if not line:
break
item = edict()
item.flag = 0
item.image_path, item.label, item.bbox, item.landmark, item.aligned = face_preprocess.parse_lst_line(line)
item.id = _id
yield item
if item.label!=last[0]:
if last[1]>=0:
identities.append( (last[1], _id) )
last[0] = item.label
last[1] = _id
_id+=1
identities.append( (last[1], _id) )
item = edict()
item.flag = 1
item.id = 0
item.label = [float(_id), float(_id+len(identities))]
yield item
for identity in identities:
item = edict()
item.flag = 2
item.id = _id
_id+=1
item.label = [float(identity[0]), float(identity[1])]
yield item
def read_list(path_in):
idToPathFile = open("id_to_path.txt", "w")
with open(path_in) as fin:
identities = []
last = [-9999999, -1]
_id = 1
baseDir = os.path.dirname(path_in)
while True:
line = fin.readline()
if not line:
break
item = edict()
item.flag = 0
item.image_path, label, item.bbox, item.landmark, item.aligned = face_preprocess.parse_lst_line(
line)
if not os.path.exists(item.image_path):
pathItemList = item.image_path.split('/')
item.image_path = os.path.join(baseDir, pathItemList[-3],
pathItemList[-2],
pathItemList[-1])
if not os.path.exists(item.image_path):
print('path error! ignore line: ', line)
continue
if not item.aligned and item.landmark is None:
print('ignore line: ', line)
continue
item.id = _id #item index
item.label = [label, item.aligned]
# record id<==>path
idToPathFile.write("%d, %s\n" % (item.id, item.image_path))
yield item
if label != last[
0]: #save (label,id ) relation info. endding of each class
if last[1] >= 0:
identities.append((last[1], _id))
last[0] = label
last[1] = _id
_id += 1
identities.append((last[1], _id))
item = edict()
item.flag = 2
item.id = 0
item.label = [float(_id), float(_id + len(identities))
] #last _id , last _id + len(ids)
yield item
for identity in identities:
item = edict()
item.flag = 2
item.id = _id
_id += 1
item.label = [float(identity[0]), float(identity[1])]
yield item # each endding item of each class
idToPathFile.close()
def read_list(path_in):
with open(path_in) as fin:
_id = 0
while True:
line = fin.readline()
if not line:
break
item = edict()
item.image_path, item.label, item.bbox, item.landmark, item.aligned = face_preprocess.parse_lst_line(
line)
item.id = _id
_id += 1
yield item
def read_list(path_in):
with open(path_in) as fin:
identities = []
last = [-1, -1] # range's [label, start id]
_id = 1 # idx == 1
while True:
line = fin.readline()
if not line:
break
item = edict()
item.flag = 0
item.image_path, label, item.bbox, item.landmark, item.aligned = face_preprocess.parse_lst_line(
line)
#if not item.aligned and item.landmark is None:
#print('ignore line', line)
#continue
item.id = _id
item.label = [label, item.aligned]
if item.bbox is not None:
item.label += list(item.bbox)
if item.landmark is not None:
item.label += item.landmark
yield item
if label != last[0]:
if last[1] >= 0:
identities.append(
(last[1], _id)) # identity range for last label
last[0] = label
last[1] = _id
_id += 1
identities.append((last[1], _id))
item = edict()
item.flag = 2
item.id = 0 # idx == 0
item.label = [float(_id), float(_id + len(identities))
] # image end idx, identity end idx
yield item
for identity in identities: # record all range
item = edict()
item.flag = 2
item.id = _id
_id += 1
item.label = [float(identity[0]), float(identity[1])]
yield item
def read_list(path_in):
with open(path_in) as fin:
identities = []
last = [-1, -1]
_id = 1
while True:
line = fin.readline()
if not line:
break
item = edict()
item.flag = 0
item.image_path, label, item.bbox, item.landmark, item.aligned = face_preprocess.parse_lst_line(line)
# item.is_card = int(path.split(item.image_path)[1][0] == 'X')
if not item.aligned and item.landmark is None:
#print('ignore line', line)
continue
item.id = _id
item.label = [label, item.aligned]
yield item
if label!=last[0]:
if last[1]>=0:
identities.append( (last[1], _id) )
last[0] = label
last[1] = _id
_id+=1
identities.append( (last[1], _id) )
item = edict()
item.flag = 2
item.id = 0
item.label = [float(_id), float(_id+len(identities))]
# item.is_card = 2
yield item
for identity in identities:
item = edict()
item.flag = 2
item.id = _id
_id+=1
item.label = [float(identity[0]), float(identity[1])]
# item.is_card = 2
yield item
def read_list(path_in):
with open(path_in) as fin:
identities = []
last = [-1, -1]
_id = 1
while True:
line = fin.readline()
if not line:
break
item = edict()
item.flag = 0
item.image_path, label, item.bbox, item.landmark, item.aligned = face_preprocess.parse_lst_line(line)
if not item.aligned and item.landmark is None:
#print('ignore line', line)
continue
item.id = _id
item.label = [label, item.aligned]
yield item
if label!=last[0]:
if last[1]>=0:
identities.append( (last[1], _id) )
last[0] = label
last[1] = _id
_id+=1
identities.append( (last[1], _id) )
item = edict()
item.flag = 2
item.id = 0
item.label = [float(_id), float(_id+len(identities))]
yield item
for identity in identities:
item = edict()
item.flag = 2
item.id = _id
_id+=1
item.label = [float(identity[0]), float(identity[1])]
yield item
def main(args):
out_algo = args.suffix
if len(args.algo) > 0:
out_algo = args.algo
fs_noise_map = {}
for line in open(args.facescrub_noises, 'r'):
if line.startswith('#'):
continue
line = line.strip()
fname = line.split('.')[0]
p = fname.rfind('_')
fname = fname[0:p]
fs_noise_map[line] = fname
print(len(fs_noise_map))
i = 0
fname2center = {}
noises = []
for line in open(args.facescrub_lst, 'r'):
if i % 1000 == 0:
print("reading fs", i)
i += 1
image_path, label, bbox, landmark, aligned = face_preprocess.parse_lst_line(
line)
assert aligned == True
_path = image_path.split('/')
a, b = _path[-2], _path[-1]
feature_path = os.path.join(args.facescrub_feature_dir, a,
"%s_%s.bin" % (b, args.suffix))
feature_dir_out = os.path.join(args.facescrub_feature_dir_out, a)
if not os.path.exists(feature_dir_out):
os.makedirs(feature_dir_out)
feature_path_out = os.path.join(args.facescrub_feature_dir_out, a,
"%s_%s.bin" % (b, out_algo))
#print(b)
if not b in fs_noise_map:
#shutil.copyfile(feature_path, feature_path_out)
feature = load_bin(feature_path)
write_bin(feature_path_out, feature)
if not a in fname2center:
fname2center[a] = np.zeros((feature_dim + feature_ext, ),
dtype=np.float32)
fname2center[a] += feature
else:
#print('n', b)
noises.append((a, b))
print(len(noises))
for k in noises:
a, b = k
assert a in fname2center
center = fname2center[a]
g = np.zeros((feature_dim + feature_ext, ), dtype=np.float32)
g2 = np.random.uniform(-0.001, 0.001, (feature_dim, ))
g[0:feature_dim] = g2
f = center + g
_norm = np.linalg.norm(f)
f /= _norm
feature_path_out = os.path.join(args.facescrub_feature_dir_out, a,
"%s_%s.bin" % (b, out_algo))
write_bin(feature_path_out, f)
mf_noise_map = {}
for line in open(args.megaface_noises, 'r'):
if line.startswith('#'):
continue
line = line.strip()
_vec = line.split("\t")
if len(_vec) > 1:
line = _vec[1]
mf_noise_map[line] = 1
print(len(mf_noise_map))
i = 0
nrof_noises = 0
for line in open(args.megaface_lst, 'r'):
if i % 1000 == 0:
print("reading mf", i)
i += 1
image_path, label, bbox, landmark, aligned = face_preprocess.parse_lst_line(
line)
assert aligned == True
_path = image_path.split('/')
a1, a2, b = _path[-3], _path[-2], _path[-1]
feature_path = os.path.join(args.megaface_feature_dir, a1, a2,
"%s_%s.bin" % (b, args.suffix))
feature_dir_out = os.path.join(args.megaface_feature_dir_out, a1, a2)
if not os.path.exists(feature_dir_out):
os.makedirs(feature_dir_out)
feature_path_out = os.path.join(args.megaface_feature_dir_out, a1, a2,
"%s_%s.bin" % (b, out_algo))
bb = '/'.join([a1, a2, b])
#print(b)
if not bb in mf_noise_map:
feature = load_bin(feature_path)
write_bin(feature_path_out, feature)
#shutil.copyfile(feature_path, feature_path_out)
else:
feature = load_bin(feature_path, 100.0)
write_bin(feature_path_out, feature)
#g = np.random.uniform(-0.001, 0.001, (feature_dim,))
#print('n', bb)
#write_bin(feature_path_out, g)
nrof_noises += 1
print(nrof_noises)
def main(args):
print(args)
gpuid = args.gpu
ctx = mx.gpu(gpuid)
nets = []
image_shape = [int(x) for x in args.image_size.split(',')]
for model in args.model.split('|'):
vec = model.split(',')
assert len(vec) > 1
prefix = vec[0]
epoch = int(vec[1])
print('loading', prefix, epoch)
net = edict()
net.ctx = ctx
net.sym, net.arg_params, net.aux_params = mx.model.load_checkpoint(prefix, epoch)
# net.arg_params, net.aux_params = ch_dev(net.arg_params, net.aux_params, net.ctx)
all_layers = net.sym.get_internals()
net.sym = all_layers['fc1_output']
net.model = mx.mod.Module(symbol=net.sym, context=net.ctx, label_names=None)
net.model.bind(data_shapes=[('data', (1, 3, image_shape[1], image_shape[2]))])
net.model.set_params(net.arg_params, net.aux_params)
# _pp = prefix.rfind('p')+1
# _pp = prefix[_pp:]
# net.patch = [int(x) for x in _pp.split('_')]
# assert len(net.patch)==5
# print('patch', net.patch)
nets.append(net)
# megaface_lst = "/raid5data/dplearn/faceinsight_align_megaface.lst"
megaface_lst = "/raid5data/dplearn/megaface/megaface_mtcnn_112x112/lst"
# facescrub_lst = "/raid5data/dplearn/faceinsight_align_facescrub.lst"
facescrub_lst = "/raid5data/dplearn/megaface/facescrubr/small_lst"
if args.fsall > 0:
facescrub_lst = "/raid5data/dplearn/megaface/facescrubr/lst"
if args.skip == 0:
i = 0
succ = 0
for line in open(facescrub_lst, 'r'):
if i % 1000 == 0:
print("writing fs", i, succ)
i += 1
image_path, label, bbox, landmark, aligned = face_preprocess.parse_lst_line(line)
_path = image_path.split('/')
a, b = _path[-2], _path[-1]
# a = a.replace(' ', '_')
# b = b.replace(' ', '_')
out_dir = os.path.join(facescrub_out, a)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
# file, ext = os.path.splitext(b)
# image_id = int(file.split('_')[-1])
# if image_id==40499 or image_id==10788 or image_id==2367:
# b = file
# if len(ext)==0:
# print(image_path)
# image_path = image_path+".jpg"
# if facescrub_aligned_root is not None:
# _vec = image_path.split('/')
# _image_path = os.path.join(facescrub_aligned_root, _vec[-2], _vec[-1])
# _base, _ext = os.path.splitext(_image_path)
# if _ext=='.gif':
# _image_path = _base+".jpg"
# print('changing', _image_path)
# if os.path.exists(_image_path):
# image_path = _image_path
# bbox = None
# landmark = None
# aligned = True
# else:
# print("not aligned:",_image_path)
feature = get_feature(image_path, bbox, landmark, nets, image_shape, True, aligned, args.mean)
if feature is None:
print('feature none', image_path)
continue
# print(np.linalg.norm(feature))
out_path = os.path.join(out_dir, b + "_%s_%dx%d.bin" % (args.algo, image_shape[1], image_shape[2]))
write_bin(out_path, feature)
succ += 1
print('fs stat', i, succ)
# return
if args.mf == 0:
return
i = 0
succ = 0
for line in open(megaface_lst, 'r'):
if i % 1000 == 0:
print("writing mf", i, succ)
i += 1
if i <= args.skip:
continue
image_path, label, bbox, landmark, aligned = face_preprocess.parse_lst_line(line)
assert aligned == True
_path = image_path.split('/')
a1, a2, b = _path[-3], _path[-2], _path[-1]
out_dir = os.path.join(megaface_out, a1, a2)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
# continue
# print(landmark)
feature = get_feature(image_path, bbox, landmark, nets, image_shape, True, aligned, args.mean)
if feature is None:
continue
out_path = os.path.join(out_dir, b + "_%s_%dx%d.bin" % (args.algo, image_shape[1], image_shape[2]))
# print(out_path)
write_bin(out_path, feature)
succ += 1
print('mf stat', i, succ)
def main(args):
out_algo = args.suffix
if len(args.algo)>0:
out_algo = args.algo
fs_noise_map = {}
for line in open(args.facescrub_noises, 'r'):
if line.startswith('#'):
continue
line = line.strip()
fname = line.split('.')[0]
p = fname.rfind('_')
fname = fname[0:p]
fs_noise_map[line] = fname
print(len(fs_noise_map))
i=0
fname2center = {}
noises = []
for line in open(args.facescrub_lst, 'r'):
if i%1000==0:
print("reading fs",i)
i+=1
image_path, label, bbox, landmark, aligned = face_preprocess.parse_lst_line(line)
assert aligned==True
_path = image_path.split('/')
a, b = _path[-2], _path[-1]
feature_path = os.path.join(args.facescrub_feature_dir, a, "%s_%s.bin"%(b, args.suffix))
feature_dir_out = os.path.join(args.facescrub_feature_dir_out, a)
if not os.path.exists(feature_dir_out):
os.makedirs(feature_dir_out)
feature_path_out = os.path.join(args.facescrub_feature_dir_out, a, "%s_%s.bin"%(b, out_algo))
#print(b)
if not b in fs_noise_map:
#shutil.copyfile(feature_path, feature_path_out)
feature = load_bin(feature_path)
write_bin(feature_path_out, feature)
if not a in fname2center:
fname2center[a] = np.zeros((feature_dim+feature_ext,), dtype=np.float32)
fname2center[a] += feature
else:
#print('n', b)
noises.append( (a,b) )
print(len(noises))
for k in noises:
a,b = k
assert a in fname2center
center = fname2center[a]
g = np.zeros( (feature_dim+feature_ext,), dtype=np.float32)
g2 = np.random.uniform(-0.001, 0.001, (feature_dim,))
g[0:feature_dim] = g2
f = center+g
_norm=np.linalg.norm(f)
f /= _norm
feature_path_out = os.path.join(args.facescrub_feature_dir_out, a, "%s_%s.bin"%(b, out_algo))
write_bin(feature_path_out, f)
mf_noise_map = {}
for line in open(args.megaface_noises, 'r'):
if line.startswith('#'):
continue
line = line.strip()
_vec = line.split("\t")
if len(_vec)>1:
line = _vec[1]
mf_noise_map[line] = 1
print(len(mf_noise_map))
i=0
nrof_noises = 0
for line in open(args.megaface_lst, 'r'):
if i%1000==0:
print("reading mf",i)
i+=1
image_path, label, bbox, landmark, aligned = face_preprocess.parse_lst_line(line)
assert aligned==True
_path = image_path.split('/')
a1, a2, b = _path[-3], _path[-2], _path[-1]
feature_path = os.path.join(args.megaface_feature_dir, a1, a2, "%s_%s.bin"%(b, args.suffix))
feature_dir_out = os.path.join(args.megaface_feature_dir_out, a1, a2)
if not os.path.exists(feature_dir_out):
os.makedirs(feature_dir_out)
feature_path_out = os.path.join(args.megaface_feature_dir_out, a1, a2, "%s_%s.bin"%(b, out_algo))
bb = '/'.join([a1, a2, b])
#print(b)
if not bb in mf_noise_map:
feature = load_bin(feature_path)
write_bin(feature_path_out, feature)
#shutil.copyfile(feature_path, feature_path_out)
else:
feature = load_bin(feature_path, 100.0)
write_bin(feature_path_out, feature)
#g = np.random.uniform(-0.001, 0.001, (feature_dim,))
#print('n', bb)
#write_bin(feature_path_out, g)
nrof_noises+=1
print(nrof_noises)
def main(args):
print(args)
gpuid = args.gpu
ctx = mx.gpu(gpuid)
nets = []
image_shape = [int(x) for x in args.image_size.split(',')]
for model in args.model.split('|'):
vec = model.split(',')
assert len(vec)>1
prefix = vec[0]
epoch = int(vec[1])
print('loading',prefix, epoch)
net = edict()
net.ctx = ctx
net.sym, net.arg_params, net.aux_params = mx.model.load_checkpoint(prefix, epoch)
#net.arg_params, net.aux_params = ch_dev(net.arg_params, net.aux_params, net.ctx)
all_layers = net.sym.get_internals()
net.sym = all_layers['fc1_output']
net.model = mx.mod.Module(symbol=net.sym, context=net.ctx, label_names = None)
net.model.bind(data_shapes=[('data', (1, 3, image_shape[1], image_shape[2]))])
net.model.set_params(net.arg_params, net.aux_params)
#_pp = prefix.rfind('p')+1
#_pp = prefix[_pp:]
#net.patch = [int(x) for x in _pp.split('_')]
#assert len(net.patch)==5
#print('patch', net.patch)
nets.append(net)
#megaface_lst = "/raid5data/dplearn/faceinsight_align_megaface.lst"
megaface_lst = "/raid5data/dplearn/megaface/megaface_mtcnn_112x112/lst"
#facescrub_lst = "/raid5data/dplearn/faceinsight_align_facescrub.lst"
facescrub_lst = "/raid5data/dplearn/megaface/facescrubr/small_lst"
if args.fsall>0:
facescrub_lst = "/raid5data/dplearn/megaface/facescrubr/lst"
if args.skip==0:
i = 0
succ = 0
for line in open(facescrub_lst, 'r'):
if i%1000==0:
print("writing fs",i, succ)
i+=1
image_path, label, bbox, landmark, aligned = face_preprocess.parse_lst_line(line)
_path = image_path.split('/')
a,b = _path[-2], _path[-1]
#a = a.replace(' ', '_')
#b = b.replace(' ', '_')
out_dir = os.path.join(facescrub_out, a)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
#file, ext = os.path.splitext(b)
#image_id = int(file.split('_')[-1])
#if image_id==40499 or image_id==10788 or image_id==2367:
# b = file
#if len(ext)==0:
# print(image_path)
# image_path = image_path+".jpg"
#if facescrub_aligned_root is not None:
# _vec = image_path.split('/')
# _image_path = os.path.join(facescrub_aligned_root, _vec[-2], _vec[-1])
# _base, _ext = os.path.splitext(_image_path)
# if _ext=='.gif':
# _image_path = _base+".jpg"
# print('changing', _image_path)
# if os.path.exists(_image_path):
# image_path = _image_path
# bbox = None
# landmark = None
# aligned = True
# else:
# print("not aligned:",_image_path)
feature = get_feature(image_path, bbox, landmark, nets, image_shape, True, aligned, args.mean)
if feature is None:
print('feature none', image_path)
continue
#print(np.linalg.norm(feature))
out_path = os.path.join(out_dir, b+"_%s_%dx%d.bin"%(args.algo, image_shape[1], image_shape[2]))
write_bin(out_path, feature)
succ+=1
print('fs stat',i, succ)
#return
if args.mf==0:
return
i = 0
succ = 0
for line in open(megaface_lst, 'r'):
if i%1000==0:
print("writing mf",i, succ)
i+=1
if i<=args.skip:
continue
image_path, label, bbox, landmark, aligned = face_preprocess.parse_lst_line(line)
assert aligned==True
_path = image_path.split('/')
a1, a2, b = _path[-3], _path[-2], _path[-1]
out_dir = os.path.join(megaface_out, a1, a2)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
#continue
#print(landmark)
feature = get_feature(image_path, bbox, landmark, nets, image_shape, True, aligned, args.mean)
if feature is None:
continue
out_path = os.path.join(out_dir, b+"_%s_%dx%d.bin"%(args.algo, image_shape[1], image_shape[2]))
#print(out_path)
write_bin(out_path, feature)
succ+=1
print('mf stat',i, succ)
