xP=[x.replace('lfw','lfw_aegan') for x in P]
xQ=[x.replace('lfw','lfw_aegan') for x in Q]
PF=model.mean_F(utils.image_feed(xP[:K],image_dims))
QF=model.mean_F(utils.image_feed(xQ[:K],image_dims))
if config.scaling=='beta':
WF=(QF-PF)/((QF-PF)**2).mean()
elif config.scaling=='none':
WF=(QF-PF)
max_iter=config.iter
init=o
# for each interpolation step
for delta in delta_params:
print(xX,b,delta)
t2=time.time()
Y=model.F_inverse(XF+WF*delta,max_iter=max_iter,initial_image=init)
t3=time.time()
print('{} minutes to reconstruct'.format((t3-t2)/60.0))
result[-1].append(Y)
max_iter=config.iter//2
init=Y
result=numpy.asarray(result)
original=numpy.asarray(original)
if 'color' in postprocess:
result=utils.color_match(numpy.expand_dims(original,1),result)
m=imageutils.montage(numpy.concatenate([numpy.expand_dims(original,1),result],axis=1))
imageutils.write('results/demo1.png',m)
print('Output is results/demo1.png')
t1=time.time()
print('{} minutes ({} minutes per image).'.format((t1-t0)/60.0,(t1-t0)/60.0/result.shape[0]/result.shape[1]))
landmarks[19] = landmarks[19]-pad_dis
landmarks[24] = landmarks[24]+pad_dis
print ("eye_dis:{},pad_dis:{}".format(eye_dis,pad_dis))
left = min(landmarks[...,1])-eye_dis
right = max(landmarks[...,1])+eye_dis
up = min(landmarks[...,0])-eye_dis
down = max(landmarks[...,0])+eye_dis
#print(im.shape)
rect = im[max(0,left):right,max(0,up):down]
#print (rect.shape)
if rect.shape[0]>h_max or rect.shape[1]>w_max:
rect = rect / 255.0
scale = min(float(h_max)/rect.shape[0],float(w_max)/rect.shape[1])
print (scale)
rect = imageutils.scale(rect,scale)
imageutils.write(postprocess_name,rect)
else:
rect = Image.fromarray(rect)
rect.save(postprocess_name)
#(x_p,y_p) = rect.size
print ("Saving image as:{}".format(postprocess_name))
# if x < width:
# w_pad = width-x
# if y < height:
# h_pad = height - y
# im = np.lib.pad(im_tmp,((w_pad//2,w_pad-w_pad//2),(h_pad//2,h_pad-h_pad//2)),'constant',constant_values=255)
# elif y > height:
# im = np.lib.pad(im_tmp,(w_pad//2,w_pad-w_pad//2),'constant',constant_values=255)
# y = y * width // x
# im_resize = im_tmp.resize((width,y),Image.ANTIALIAS)
# new_name = root +'new_'+imname
max_iter=max_iter,
initial_image=init)
max_iter = config.step_iter
init = result
i = i + 1
delta = delta + ddelta
if 'mask' in postprocess and os.path.exists(X_mask):
result *= mask
result += original * (1 - mask)
if 'color' in postprocess:
result = utils.color_match(numpy.asarray([original]),
numpy.asarray([[result]]))[0, 0]
if 'mask' in postprocess and os.path.exists(X_mask):
result *= mask
result += original * (1 - mask)
imageutils.write(
prefix_path + postfix_comment + '/{:06}.png'.format(i), result)
# generate movie
cmd = [
'ffmpeg', '-y', '-f', 'image2', '-i',
prefix_path + postfix_comment + '/%06d.png', '-crf', '19', '-g', '60',
'-r', '30', '-s', '{}x{}'.format(original.shape[1], original.shape[0]),
'-pix_fmt', 'yuv420p', prefix_path + postfix_comment +
'_movie.{}'.format(config.output_format)
]
print(' '.join(pipes.quote(x) for x in cmd))
subprocess.check_call(cmd)
print('Output is {}'.format(prefix_path + postfix_comment +
'_movie.{}'.format(config.output_format)))
print('{} minutes to reconstruct'.format((t3-t2)/60.0))
result.append(Y)
max_iter=config.iter//2
init=Y
result=numpy.asarray([result])
original=numpy.asarray([original])
X_mask=prefix_path+'-mask.png'
if 'mask' in postprocess and os.path.exists(X_mask):
mask=imageutils.resize(imageutils.read(X_mask),image_dims)
result*=mask
result+=original*(1-mask)
if 'color' in postprocess:
result=utils.color_match(numpy.asarray([original]),result)
if 'mask' in postprocess and os.path.exists(X_mask):
result*=mask
result+=original*(1-mask)
if config.include_original:
m=imageutils.montage(numpy.concatenate([numpy.expand_dims(original,1),result],axis=1))
else:
m=imageutils.montage(result)
if config.output:
opath=config.output
else:
opath='{}_{}_{}{}.{}'.format(prefix_path,timestamp,config.method,postfix_comment,config.output_format)
imageutils.write(opath,m)
opathlist.append(opath)
print('Outputs are {}'.format(' '.join(opathlist)))
t1=time.time()
print('{} minutes ({} minutes per image).'.format((t1-t0)/60.0,(t1-t0)/60.0/len(X)/len(delta_params)))
def main(config):
caffe.set_device(config['device_id'])
caffe.set_mode_gpu()
# step 1. configure cnn
if config['arch']=='A4,G1':
# an ensemble of alexnet and googlenet
models=[
caffe.Net('deploy-alexnet_full_conv.prototxt','minc-alexnet_full_conv.caffemodel',caffe.TEST),
caffe.Net('deploy-googlenet_full_conv_no_pooling.prototxt','minc-googlenet_full_conv.caffemodel',caffe.TEST)
]
# alexnet needs a padded input to get a full-frame prediction
input_padding=[97,0]
# nominal footprint is 46.4% for A4, 23.2% for G1
scales=[256/550.0,256/1100.0]
bgr_mean=numpy.array([104,117,124],dtype=numpy.float32)
# inputs must be a multiple of the stride
# otherwise, the full-frame prediction will be shifted
effective_stride=[32,32]
# TODO: A4 needs spatial oversampling (# shifts = 2)
else:
raise NotImplementedError
# step 2. configure crf
if config['crf']=='1':
# these are the CRF parameters for MINC (see supplemental)
# the parameters can have a big impact on the output
# so they should be tuned for the target domain
# (MINC note: this code is not exactly the same as the
# original MINC code so these parameters will not
# generate the exact same results)
crf_params={
"bilateral_pairwise_weight": 5.0, # w_p
"bilateral_theta_xy": 0.1, # \theta_p
"bilateral_theta_lab_l": 20.0, # \theta_L
"bilateral_theta_lab_ab": 5.0, # \theta_ab
"n_crf_iters": 10,
"unary_prob_padding": 1e-05,
}
elif config['crf']=='matclass':
# new CRF parameters
crf_params={
"bilateral_pairwise_weight": 8.0, # w_p
"bilateral_theta_xy": 0.5, # \theta_p
"bilateral_theta_lab_l": 0.5, # \theta_L
"bilateral_theta_lab_ab": 3.0, # \theta_ab
"n_crf_iters": 10,
"unary_prob_padding": 1e-05,
}
else:
raise NotImplementedError
pad_value=bgr_mean[::-1]/255.0
# step 3. extract class prediction maps
for ipath in config['input']:
# read image
original=imageutils.read(ipath)
z=config['min_dim']/float(min(original.shape[:2]))
crf_shape=(23,int(round(original.shape[0]*z)),int(round(original.shape[1]*z)))
# predict 6 maps: 3 scales for each model
maps=[]
for index,model in enumerate(models):
p=input_padding[index]
s=scales[index]
for index2,multiscale in enumerate([0.7071067811865476,1.0,1.4142135623730951]):
# resample the input so it is a multiple of the stride
# and the receptive field matches the nominal footprint
scale_factor=(256/s)/float(min(original.shape[:2]))
scaled_size=[nearest_multiple(original.shape[i]*scale_factor*multiscale,effective_stride[index]) for i in range(2)]
scaled=imageutils.resize(original,scaled_size)
if p>0:
# add input padding for alexnet
pad=numpy.ones((scaled.shape[0]+2*p,scaled.shape[1]+2*p,scaled.shape[2]),dtype=scaled.dtype)*pad_value
pad[p:-p,p:-p]=scaled
scaled=pad
# predict and resample the map to be the correct size
data=preprocess_and_reshape(scaled,model,bgr_mean=bgr_mean)
output=model.forward_all(data=data)['prob'][0]
output=scipy.ndimage.interpolation.zoom(output,[1.0,crf_shape[1]/float(output.shape[1]),crf_shape[2]/float(output.shape[2])],order=1)
maps.append(output)
# step 4. average all maps
crf_map=numpy.asarray(maps).mean(axis=0)
if False:
# output extra maps for debugging
for i,x,j in [(i,x,j) for i,x in enumerate(maps) for j in range(23)]: imageutils.write('zzz_map_{}{}.jpg'.format(dataset.NETCAT_TO_NAME[j],i),x[j])
for j in range(23): imageutils.write('zzz_mean_map_{}.jpg'.format(dataset.NETCAT_TO_NAME[j],i),crf_map[j])
imageutils.write('zzz_naive_labels.png',labels_to_color(numpy.argsort(-crf_map.reshape(23,-1).T).T.reshape(*crf_map.shape)[0]))
crf_color=imageutils.resize(original,(crf_shape[1],crf_shape[2]))
assert crf_color.shape[0]==crf_map.shape[1] and crf_color.shape[1]==crf_map.shape[2]
# step 5. dense crf
#lcrf=densecrf_matclass.general_densecrf.LearnableDenseCRF(crf_color,crf_map,crf_params)
lcrf=densecrf_matclass.densecrf.LearnableDenseCRF(crf_color,crf_map,crf_params)
labels_crf=lcrf.map(crf_params)
# step 6. visualize with color labels
result=labels_to_color(labels_crf)
if os.path.exists(config['output']) and os.path.isdir(config['output']):
opath=os.path.join(config['output'],os.path.splitext(os.path.split(ipath)[1])[0])+'.png'
else:
opath=config['output']
assert not os.path.exists(opath)
imageutils.write(opath,result)
print(opath)