def warped_image_feed(S, MP, image_dims):
'''
Given a list of file paths, warp matrices and a 2-tuple of (H, W),
yields H x W x 3 images.
'''
for i, x in enumerate(S):
I = imageutils.read(x)
yield numpy.asarray(
alignface.warp_to_template(I, MP[i], image_dims=image_dims))
def image_feed(S, image_dims):
'''
Given a list of file paths and a 2-tuple of (H, W), yields H x W x 3 images.
'''
for x in S:
I = imageutils.read(x)
if I.shape[:2] != image_dims:
yield imageutils.resize(I, tuple(image_dims))
else:
yield I
def image_feed_masked(S, image_dims):
'''
Given a list of file paths and a 2-tuple of (H, W), yields H x W x 3 images.
'''
for x in S:
I = imageutils.read(x)
if I.shape[:2] != image_dims:
I = imageutils.resize(I, tuple(image_dims))
center_mask_inplace(I)
yield I
# comment out the line below to generate all the test images
X=X[:1]
# Set the free parameters
# Note: for LFW, 0.4*8.82 is approximately equivalent to beta=0.4
K=config.K
delta_params=[float(x.strip()) for x in config.delta.split(',')]
t0=time.time()
result=[]
original=[]
# for each test image
for i in range(len(X)):
result.append([])
xX=X[i].replace('lfw','lfw_aegan')
o=imageutils.read(xX)
image_dims=o.shape[:2]
if min(image_dims)<minimum_resolution:
s=float(minimum_resolution)/min(image_dims)
image_dims=(int(round(image_dims[0]*s)),int(round(image_dims[1]*s)))
o=imageutils.resize(o,image_dims)
XF=model.mean_F([o])
original.append(o)
# for each transform
for j,(a,b) in enumerate(pairs):
_,P,Q=make_manifolds(b,[a],[],X=X[i:i+1],N=1)
P=P[0]
Q=Q[0]
xP=[x.replace('lfw','lfw_aegan') for x in P]
xQ=[x.replace('lfw','lfw_aegan') for x in Q]
PF=model.mean_F(utils.image_feed(xP[:K],image_dims))
# for each interpolation step
result=[]
for delta in delta_params: #每一个delta
print(xX,image_dims,delta,len(xP),len(xQ))
t2=time.time()
Y=model.F_inverse(XF+WF*delta,max_iter=max_iter,initial_image=init)
t3=time.time()
print('{} minutes to reconstruct'.format((t3-t2)/60.0))
result.append(Y)
max_iter=config.iter//2
init=Y
result=numpy.asarray([result])
original=numpy.asarray([original])
X_mask=prefix_path+'-mask.png'
if 'mask' in postprocess and os.path.exists(X_mask):
mask=imageutils.resize(imageutils.read(X_mask),image_dims)
result*=mask
result+=original*(1-mask)
if 'color' in postprocess:
result=utils.color_match(numpy.asarray([original]),result)
if 'mask' in postprocess and os.path.exists(X_mask):
result*=mask
result+=original*(1-mask)
if config.include_original:
m=imageutils.montage(numpy.concatenate([numpy.expand_dims(original,1),result],axis=1))
else:
m=imageutils.montage(result)
if config.output:
opath=config.output
else:
opath='{}_{}_{}{}.{}'.format(prefix_path,timestamp,config.method,postfix_comment,config.output_format)
assert 0 <= delta0
assert delta0 <= delta1
ddelta = config.delta_step
assert ddelta > 0
# load models
if config.backend == 'torch':
import deepmodels_torch
model = deepmodels_torch.vgg19g_torch(device_id=config.device_id)
elif config.backend == 'caffe+scipy':
model = deepmodels.vgg19g(device_id=config.device_id)
else:
raise ValueError('Unknown backend')
# load interpolation data
original = imageutils.read(ipath1)
XF = model.mean_F([original])
prefix_path = os.path.splitext(ipath1)[0]
if not os.path.exists(prefix_path + postfix_comment):
os.mkdir(prefix_path + postfix_comment) # debug
X_mask = prefix_path + '-mask.png'
if 'mask' in postprocess and os.path.exists(X_mask):
mask = imageutils.resize(imageutils.read(X_mask), image_dims)
data = numpy.load(ipath2)
if 'WF' in data:
WF = data['WF']
elif 'muQ' in data and 'muP' in data:
WF = data['muQ'] - data['muP']
# generate frames
t0 = time.time()
def main(config):
caffe.set_device(config['device_id'])
caffe.set_mode_gpu()
# step 1. configure cnn
if config['arch']=='A4,G1':
# an ensemble of alexnet and googlenet
models=[
caffe.Net('deploy-alexnet_full_conv.prototxt','minc-alexnet_full_conv.caffemodel',caffe.TEST),
caffe.Net('deploy-googlenet_full_conv_no_pooling.prototxt','minc-googlenet_full_conv.caffemodel',caffe.TEST)
]
# alexnet needs a padded input to get a full-frame prediction
input_padding=[97,0]
# nominal footprint is 46.4% for A4, 23.2% for G1
scales=[256/550.0,256/1100.0]
bgr_mean=numpy.array([104,117,124],dtype=numpy.float32)
# inputs must be a multiple of the stride
# otherwise, the full-frame prediction will be shifted
effective_stride=[32,32]
# TODO: A4 needs spatial oversampling (# shifts = 2)
else:
raise NotImplementedError
# step 2. configure crf
if config['crf']=='1':
# these are the CRF parameters for MINC (see supplemental)
# the parameters can have a big impact on the output
# so they should be tuned for the target domain
# (MINC note: this code is not exactly the same as the
# original MINC code so these parameters will not
# generate the exact same results)
crf_params={
"bilateral_pairwise_weight": 5.0, # w_p
"bilateral_theta_xy": 0.1, # \theta_p
"bilateral_theta_lab_l": 20.0, # \theta_L
"bilateral_theta_lab_ab": 5.0, # \theta_ab
"n_crf_iters": 10,
"unary_prob_padding": 1e-05,
}
elif config['crf']=='matclass':
# new CRF parameters
crf_params={
"bilateral_pairwise_weight": 8.0, # w_p
"bilateral_theta_xy": 0.5, # \theta_p
"bilateral_theta_lab_l": 0.5, # \theta_L
"bilateral_theta_lab_ab": 3.0, # \theta_ab
"n_crf_iters": 10,
"unary_prob_padding": 1e-05,
}
else:
raise NotImplementedError
pad_value=bgr_mean[::-1]/255.0
# step 3. extract class prediction maps
for ipath in config['input']:
# read image
original=imageutils.read(ipath)
z=config['min_dim']/float(min(original.shape[:2]))
crf_shape=(23,int(round(original.shape[0]*z)),int(round(original.shape[1]*z)))
# predict 6 maps: 3 scales for each model
maps=[]
for index,model in enumerate(models):
p=input_padding[index]
s=scales[index]
for index2,multiscale in enumerate([0.7071067811865476,1.0,1.4142135623730951]):
# resample the input so it is a multiple of the stride
# and the receptive field matches the nominal footprint
scale_factor=(256/s)/float(min(original.shape[:2]))
scaled_size=[nearest_multiple(original.shape[i]*scale_factor*multiscale,effective_stride[index]) for i in range(2)]
scaled=imageutils.resize(original,scaled_size)
if p>0:
# add input padding for alexnet
pad=numpy.ones((scaled.shape[0]+2*p,scaled.shape[1]+2*p,scaled.shape[2]),dtype=scaled.dtype)*pad_value
pad[p:-p,p:-p]=scaled
scaled=pad
# predict and resample the map to be the correct size
data=preprocess_and_reshape(scaled,model,bgr_mean=bgr_mean)
output=model.forward_all(data=data)['prob'][0]
output=scipy.ndimage.interpolation.zoom(output,[1.0,crf_shape[1]/float(output.shape[1]),crf_shape[2]/float(output.shape[2])],order=1)
maps.append(output)
# step 4. average all maps
crf_map=numpy.asarray(maps).mean(axis=0)
if False:
# output extra maps for debugging
for i,x,j in [(i,x,j) for i,x in enumerate(maps) for j in range(23)]: imageutils.write('zzz_map_{}{}.jpg'.format(dataset.NETCAT_TO_NAME[j],i),x[j])
for j in range(23): imageutils.write('zzz_mean_map_{}.jpg'.format(dataset.NETCAT_TO_NAME[j],i),crf_map[j])
imageutils.write('zzz_naive_labels.png',labels_to_color(numpy.argsort(-crf_map.reshape(23,-1).T).T.reshape(*crf_map.shape)[0]))
crf_color=imageutils.resize(original,(crf_shape[1],crf_shape[2]))
assert crf_color.shape[0]==crf_map.shape[1] and crf_color.shape[1]==crf_map.shape[2]
# step 5. dense crf
#lcrf=densecrf_matclass.general_densecrf.LearnableDenseCRF(crf_color,crf_map,crf_params)
lcrf=densecrf_matclass.densecrf.LearnableDenseCRF(crf_color,crf_map,crf_params)
labels_crf=lcrf.map(crf_params)
# step 6. visualize with color labels
result=labels_to_color(labels_crf)
if os.path.exists(config['output']) and os.path.isdir(config['output']):
opath=os.path.join(config['output'],os.path.splitext(os.path.split(ipath)[1])[0])+'.png'
else:
opath=config['output']
assert not os.path.exists(opath)
imageutils.write(opath,result)
print(opath)