def view_combined_model(shp_fn, txt_fn, cmb_fn, scale, tranx, trany, width, height):
img = np.empty((height, width, 3), dtype='uint8')
# cv2.namedWindow('combined model')
tm = TextureMapper(img.shape[1], img.shape[0])
smodel = ShapeModel.load(shp_fn)
tmodel = TextureModel.load(txt_fn)
cmodel = CombinedModel.load(cmb_fn)
vals = genvals()
params = smodel.get_params(scale, tranx, trany)
ref = smodel.calc_shape(params)
ref = ref.reshape((ref.size // 2, 2))
verts = get_vertices(ref)
while True:
for k in range(cmodel.num_modes()):
for v in vals:
p = np.zeros(cmodel.num_modes())
p[k] = v * 3 * np.sqrt(cmodel.variance[k])
sparams, tparams = cmodel.calc_shp_tex_params(p, smodel.num_modes())
params[4:] = sparams
shp = smodel.calc_shape(params)
shp = shp.reshape(ref.shape)
t = tmodel.calc_texture(tparams)
img[:] = 0
draw_texture(img, t, ref)
warped = tm.warp_triangles(img, ref[verts], shp[verts])
s = 'mode: %d, val: %f sd' % (k, v * 3)
draw_string(warped, s)
print('#####')
# cv2.imshow('combined model', warped)
cv2.imwrite('output/%d.jpg'%(int(time.time()*1000000)), warped)
def view_combined_model(shp_fn, txt_fn, cmb_fn, scale, tranx, trany, width, height):
img = np.empty((height, width, 3), dtype='uint8')
cv2.namedWindow('combined model')
tm = TextureMapper(img.shape[1], img.shape[0])
smodel = ShapeModel.load(shp_fn)
tmodel = TextureModel.load(txt_fn)
cmodel = CombinedModel.load(cmb_fn)
vals = genvals()
params = smodel.get_params(scale, tranx, trany)
ref = smodel.calc_shape(params)
ref = ref.reshape((ref.size//2, 2))
verts = get_vertices(ref)
while True:
for k in xrange(cmodel.num_modes()):
for v in vals:
p = np.zeros(cmodel.num_modes())
p[k] = v * 3 * np.sqrt(cmodel.variance[k])
sparams, tparams = cmodel.calc_shp_tex_params(p, smodel.num_modes())
params[4:] = sparams
shp = smodel.calc_shape(params)
shp = shp.reshape(ref.shape)
t = tmodel.calc_texture(tparams)
img[:] = 0
draw_texture(img, t, ref)
warped = tm.warp_triangles(img, ref[verts], shp[verts])
s = 'mode: %d, val: %f sd' % (k, v*3)
draw_string(warped, s)
cv2.imshow('combined model', warped)
if cv2.waitKey(10) == 27:
sys.exit()
def get_data_matrix(imgs, lmks, ref):
ref = ref.reshape((ref.size//2, 2)).astype('int32')
mask = get_mask(ref, (640, 480)) # FIXME hardcoded image size
verts = get_vertices(ref)
tm = TextureMapper(480, 640) # ditto
n_samples = lmks.shape[1]
n_pixels = mask.sum() * 3
G = np.empty((n_pixels, n_samples))
for i in range(n_samples):
src = lmks[:,i].reshape(ref.shape)
img = normalize(next(imgs), get_aabb(src))
warp = tm.warp_triangles(img, src[verts], ref[verts])
G[:,i] = warp[mask].ravel()
return G
def get_data_matrix(imgs, lmks, ref):
ref = ref.reshape((ref.size//2, 2)).astype('int32')
mask = get_mask(ref, (640, 480)) # FIXME hardcoded image size
verts = get_vertices(ref)
tm = TextureMapper(480, 640) # ditto
n_samples = lmks.shape[1]
n_pixels = mask.sum() * 3
G = np.empty((n_pixels, n_samples))
for i in xrange(n_samples):
src = lmks[:,i].reshape(ref.shape)
img = normalize(next(imgs), get_aabb(src))
warp = tm.warp_triangles(img, src[verts], ref[verts])
G[:,i] = warp[mask].ravel()
return G
def train(cls, lmks, imgs, ref, frac, kmax):
smodel = ShapeModel.load('data/shape.npz')
tmodel = TextureModel.load('data/texture.npz')
# build diagonal matrix of weights that measures
# the unit difference between shape and texture parameters
r = tmodel.variance.sum() / smodel.variance.sum()
Ws = r * np.eye(smodel.num_modes())
n_samples = lmks.shape[1]
tm = TextureMapper(480, 640)
# create empty matrix
n_feats = smodel.num_modes() + tmodel.num_modes()
A = np.empty((n_feats, n_samples))
# concatenate shape and texture parameters for each training example
for i in xrange(n_samples):
img = next(imgs)
lmk = lmks[:, i]
sparams = smodel.calc_params(lmk)
tparams = tmodel.calc_params(img, lmk, ref, tm.warp_triangles)
# ignore first 4 shape parameters
A[:, i] = np.concatenate((Ws.dot(sparams[4:]), tparams))
D = pca(A, frac, kmax)
# compute variance
Q = pow(D.T.dot(A), 2)
e = Q.sum(axis=1) / (n_samples - 1)
return cls(D, e, Ws)
def experiments(images, landmarks, smodel, tmodel, ref_shape, fout):
tm = TextureMapper(480, 640)
tri = get_vertices(ref_shape)
split = smodel.num_modes() + 4
perturbator = Perturbator(np.sqrt(smodel.variance[4:]), np.sqrt(tmodel.variance))
n_samples = len(landmarks)
n_perts = perturbator.num_perts()
total_perts = n_perts * n_samples
n_params = 4 + smodel.num_modes() + tmodel.num_modes()
t_vec_sz = tmodel.texture_vector_size()
h5 = tb.openFile(fout, mode='w', title='perturbations')
filters = tb.Filters(complevel=5, complib='blosc')
P = h5.createCArray(h5.root, 'perturbations', tb.Float64Atom(),
shape=(n_params, total_perts), filters=filters)
R = h5.createCArray(h5.root, 'residuals', tb.Float64Atom(),
shape=(t_vec_sz, total_perts), filters=filters,
chunkshape=(2048, 128))
for i in xrange(len(landmarks)):
# get image and corresponding landmarks
img = next(images)
lmks = landmarks[i]
pts = lmks.reshape(ref_shape.shape)
# get shape and texture model parameters for current example
s_params = smodel.calc_params(lmks)
t_params = tmodel.calc_params(img, lmks, ref_shape, tm.warp_triangles)
params = np.concatenate((s_params, t_params))
perturbations = perturbator.perturbations(s_params, t_params)
for j,pert in enumerate(perturbations):
col = n_perts * i + j
print 'perturbation {:,} of {:,}'.format(col+1, total_perts)
s = pert[:split]
t = pert[split:]
x_image = smodel.calc_shape(s)
x_image = x_image.reshape((x_image.size//2, 2))
g_image = sample_texture(img, x_image, ref_shape, tm)
g_model = tmodel.calc_texture(t)
perturbation = pert - params
residual = g_image - g_model
P[:,col] = perturbation
R[:,col] = residual
h5.close()
def experiments(images, landmarks, smodel, tmodel, ref_shape):
cv2.namedWindow('original')
cv2.namedWindow('model')
cv2.namedWindow('perturbed')
tm = TextureMapper(480, 640)
tri = get_vertices(ref_shape)
split = smodel.num_modes() + 4
perturbator = Perturbator(np.sqrt(smodel.variance[4:]), np.sqrt(tmodel.variance))
for i in xrange(len(landmarks)):
# get image and corresponding landmarks
img = next(images)
lmks = landmarks[i]
pts = lmks.reshape(ref_shape.shape)
# get shape and texture model parameters for current example
s_params = smodel.calc_params(lmks)
t_params = tmodel.calc_params(img, lmks, ref_shape, tm.warp_triangles)
params = np.concatenate((s_params, t_params))
cv2.imshow('original', img)
shape = smodel.calc_shape(s_params)
shape = shape.reshape((shape.size//2, 2))
texture = tmodel.calc_texture(t_params)
warped = draw_face(img, shape, texture, ref_shape, tm.warp_triangles)
cv2.imshow('model', warped)
for pert in perturbator.perturbations(s_params, t_params):
s = pert[:split]
t = pert[split:]
x_image = smodel.calc_shape(s)
x_image = x_image.reshape((x_image.size//2, 2))
g_image = sample_texture(img, x_image, ref_shape, tm.warp_triangles)
g_model = tmodel.calc_texture(t)
perturbation = pert - params
residual = g_image - g_model
warped = draw_face(img, x_image, g_model, ref_shape, tm.warp_triangles)
cv2.imshow('perturbed', warped)
key = cv2.waitKey()
if key == ord('n'):
break
elif key == 27:
sys.exit()
cv2.namedWindow('triangles')
cv2.namedWindow('warp')
smodel = ShapeModel.load('data/shape.npz')
params = smodel.get_params(200, 150, 150)
ref = smodel.calc_shape(params)
ref = ref.reshape((len(ref) // 2, 2))
verts = get_vertices(ref)
muct = MuctDataset()
muct.load(clean=True)
if args.use_texturemapper:
from pyaam.texturemapper import TextureMapper
tm = TextureMapper(480, 640)
warp_triangles = tm.warp_triangles
for name, tag, lmks, flipped in muct.iterdata():
img = muct.image(name)
pts = lmks.reshape((len(lmks) // 2, 2))
aabb = get_aabb(pts)
if args.normalize:
img = normalize(img, aabb)
orig = img.copy()
if args.show_triangles:
draw_polygons(img, pts[verts], Color.blue)
cv2.imshow('triangles', img)
warped = warp_triangles(orig, pts[verts], ref[verts])
if args.show_triangles:
draw_polygons(warped, ref[verts], Color.blue)
def test_aam(images, landmarks, smodel, tmodel, R, ref_shape):
cv2.namedWindow('original')
cv2.namedWindow('fitted')
cv2.namedWindow('shape')
tm = TextureMapper(480, 640)
tri = get_vertices(ref_shape)
for i in xrange(len(landmarks)):
img = next(images)
cv2.imshow('original', img)
lmks = landmarks[i].reshape(ref_shape.shape)
# detect face
pts = detector.detect(img)
# get params for detected face shape
s_params = smodel.calc_params(pts)
# mean texture
t_params = np.zeros(tmodel.num_modes())
# concatenate parameters
params = np.concatenate((s_params, t_params))
shape, texture = get_instance(params, smodel, tmodel)
warped = draw_face(img, shape, texture, ref_shape, tm.warp_triangles)
cv2.imshow('fitted', warped)
img2 = img.copy()
draw_muct_shape(img2, shape.ravel())
cv2.imshow('shape', img2)
key = cv2.waitKey()
if key == ord(' '):
pass
elif key == ord('n'):
continue
elif key == 27:
sys.exit()
shape, texture = get_instance(params, smodel, tmodel)
g_image = sample_texture(img, shape, ref_shape, tm.warp_triangles)
# compute residual
residual = g_image - texture
# evaluate error
E0 = np.dot(residual, residual)
# predict model displacements
pert = R.dot(residual)
for i in xrange(MAX_ITER):
shape, texture = get_instance(params, smodel, tmodel)
g_image = sample_texture(img, shape, ref_shape, tm.warp_triangles)
# compute residual
residual = g_image - texture
# predict model displacements
pert = R.dot(residual)
for alpha in (1.5, 1, 0.5, 0.25, 0.125):
new_params = params - alpha * pert
shape, texture = get_instance(new_params, smodel, tmodel)
g_image = sample_texture(img, shape, ref_shape,
tm.warp_triangles)
residual = g_image - texture
Ek = np.dot(residual, residual)
if Ek < E0:
params = new_params
E0 = Ek
break
shape, texture = get_instance(params, smodel, tmodel)
warped = draw_face(img, shape, texture, ref_shape,
tm.warp_triangles)
cv2.imshow('fitted', warped)
img2 = img.copy()
draw_muct_shape(img2, shape.ravel())
cv2.imshow('shape', img2)
key = cv2.waitKey()
if key == ord(' '):
continue
elif key == ord('n'):
break
elif key == 27:
sys.exit()