def view_combined_model(shp_fn, txt_fn, cmb_fn, scale, tranx, trany, width, height): img = np.empty((height, width, 3), dtype='uint8') # cv2.namedWindow('combined model') tm = TextureMapper(img.shape[1], img.shape[0]) smodel = ShapeModel.load(shp_fn) tmodel = TextureModel.load(txt_fn) cmodel = CombinedModel.load(cmb_fn) vals = genvals() params = smodel.get_params(scale, tranx, trany) ref = smodel.calc_shape(params) ref = ref.reshape((ref.size // 2, 2)) verts = get_vertices(ref) while True: for k in range(cmodel.num_modes()): for v in vals: p = np.zeros(cmodel.num_modes()) p[k] = v * 3 * np.sqrt(cmodel.variance[k]) sparams, tparams = cmodel.calc_shp_tex_params(p, smodel.num_modes()) params[4:] = sparams shp = smodel.calc_shape(params) shp = shp.reshape(ref.shape) t = tmodel.calc_texture(tparams) img[:] = 0 draw_texture(img, t, ref) warped = tm.warp_triangles(img, ref[verts], shp[verts]) s = 'mode: %d, val: %f sd' % (k, v * 3) draw_string(warped, s) print('#####') # cv2.imshow('combined model', warped) cv2.imwrite('output/%d.jpg'%(int(time.time()*1000000)), warped)
def view_combined_model(shp_fn, txt_fn, cmb_fn, scale, tranx, trany, width, height): img = np.empty((height, width, 3), dtype='uint8') cv2.namedWindow('combined model') tm = TextureMapper(img.shape[1], img.shape[0]) smodel = ShapeModel.load(shp_fn) tmodel = TextureModel.load(txt_fn) cmodel = CombinedModel.load(cmb_fn) vals = genvals() params = smodel.get_params(scale, tranx, trany) ref = smodel.calc_shape(params) ref = ref.reshape((ref.size//2, 2)) verts = get_vertices(ref) while True: for k in xrange(cmodel.num_modes()): for v in vals: p = np.zeros(cmodel.num_modes()) p[k] = v * 3 * np.sqrt(cmodel.variance[k]) sparams, tparams = cmodel.calc_shp_tex_params(p, smodel.num_modes()) params[4:] = sparams shp = smodel.calc_shape(params) shp = shp.reshape(ref.shape) t = tmodel.calc_texture(tparams) img[:] = 0 draw_texture(img, t, ref) warped = tm.warp_triangles(img, ref[verts], shp[verts]) s = 'mode: %d, val: %f sd' % (k, v*3) draw_string(warped, s) cv2.imshow('combined model', warped) if cv2.waitKey(10) == 27: sys.exit()
def get_data_matrix(imgs, lmks, ref): ref = ref.reshape((ref.size//2, 2)).astype('int32') mask = get_mask(ref, (640, 480)) # FIXME hardcoded image size verts = get_vertices(ref) tm = TextureMapper(480, 640) # ditto n_samples = lmks.shape[1] n_pixels = mask.sum() * 3 G = np.empty((n_pixels, n_samples)) for i in range(n_samples): src = lmks[:,i].reshape(ref.shape) img = normalize(next(imgs), get_aabb(src)) warp = tm.warp_triangles(img, src[verts], ref[verts]) G[:,i] = warp[mask].ravel() return G
def get_data_matrix(imgs, lmks, ref): ref = ref.reshape((ref.size//2, 2)).astype('int32') mask = get_mask(ref, (640, 480)) # FIXME hardcoded image size verts = get_vertices(ref) tm = TextureMapper(480, 640) # ditto n_samples = lmks.shape[1] n_pixels = mask.sum() * 3 G = np.empty((n_pixels, n_samples)) for i in xrange(n_samples): src = lmks[:,i].reshape(ref.shape) img = normalize(next(imgs), get_aabb(src)) warp = tm.warp_triangles(img, src[verts], ref[verts]) G[:,i] = warp[mask].ravel() return G
def train(cls, lmks, imgs, ref, frac, kmax): smodel = ShapeModel.load('data/shape.npz') tmodel = TextureModel.load('data/texture.npz') # build diagonal matrix of weights that measures # the unit difference between shape and texture parameters r = tmodel.variance.sum() / smodel.variance.sum() Ws = r * np.eye(smodel.num_modes()) n_samples = lmks.shape[1] tm = TextureMapper(480, 640) # create empty matrix n_feats = smodel.num_modes() + tmodel.num_modes() A = np.empty((n_feats, n_samples)) # concatenate shape and texture parameters for each training example for i in xrange(n_samples): img = next(imgs) lmk = lmks[:, i] sparams = smodel.calc_params(lmk) tparams = tmodel.calc_params(img, lmk, ref, tm.warp_triangles) # ignore first 4 shape parameters A[:, i] = np.concatenate((Ws.dot(sparams[4:]), tparams)) D = pca(A, frac, kmax) # compute variance Q = pow(D.T.dot(A), 2) e = Q.sum(axis=1) / (n_samples - 1) return cls(D, e, Ws)
def experiments(images, landmarks, smodel, tmodel, ref_shape, fout): tm = TextureMapper(480, 640) tri = get_vertices(ref_shape) split = smodel.num_modes() + 4 perturbator = Perturbator(np.sqrt(smodel.variance[4:]), np.sqrt(tmodel.variance)) n_samples = len(landmarks) n_perts = perturbator.num_perts() total_perts = n_perts * n_samples n_params = 4 + smodel.num_modes() + tmodel.num_modes() t_vec_sz = tmodel.texture_vector_size() h5 = tb.openFile(fout, mode='w', title='perturbations') filters = tb.Filters(complevel=5, complib='blosc') P = h5.createCArray(h5.root, 'perturbations', tb.Float64Atom(), shape=(n_params, total_perts), filters=filters) R = h5.createCArray(h5.root, 'residuals', tb.Float64Atom(), shape=(t_vec_sz, total_perts), filters=filters, chunkshape=(2048, 128)) for i in xrange(len(landmarks)): # get image and corresponding landmarks img = next(images) lmks = landmarks[i] pts = lmks.reshape(ref_shape.shape) # get shape and texture model parameters for current example s_params = smodel.calc_params(lmks) t_params = tmodel.calc_params(img, lmks, ref_shape, tm.warp_triangles) params = np.concatenate((s_params, t_params)) perturbations = perturbator.perturbations(s_params, t_params) for j,pert in enumerate(perturbations): col = n_perts * i + j print 'perturbation {:,} of {:,}'.format(col+1, total_perts) s = pert[:split] t = pert[split:] x_image = smodel.calc_shape(s) x_image = x_image.reshape((x_image.size//2, 2)) g_image = sample_texture(img, x_image, ref_shape, tm) g_model = tmodel.calc_texture(t) perturbation = pert - params residual = g_image - g_model P[:,col] = perturbation R[:,col] = residual h5.close()
def experiments(images, landmarks, smodel, tmodel, ref_shape): cv2.namedWindow('original') cv2.namedWindow('model') cv2.namedWindow('perturbed') tm = TextureMapper(480, 640) tri = get_vertices(ref_shape) split = smodel.num_modes() + 4 perturbator = Perturbator(np.sqrt(smodel.variance[4:]), np.sqrt(tmodel.variance)) for i in xrange(len(landmarks)): # get image and corresponding landmarks img = next(images) lmks = landmarks[i] pts = lmks.reshape(ref_shape.shape) # get shape and texture model parameters for current example s_params = smodel.calc_params(lmks) t_params = tmodel.calc_params(img, lmks, ref_shape, tm.warp_triangles) params = np.concatenate((s_params, t_params)) cv2.imshow('original', img) shape = smodel.calc_shape(s_params) shape = shape.reshape((shape.size//2, 2)) texture = tmodel.calc_texture(t_params) warped = draw_face(img, shape, texture, ref_shape, tm.warp_triangles) cv2.imshow('model', warped) for pert in perturbator.perturbations(s_params, t_params): s = pert[:split] t = pert[split:] x_image = smodel.calc_shape(s) x_image = x_image.reshape((x_image.size//2, 2)) g_image = sample_texture(img, x_image, ref_shape, tm.warp_triangles) g_model = tmodel.calc_texture(t) perturbation = pert - params residual = g_image - g_model warped = draw_face(img, x_image, g_model, ref_shape, tm.warp_triangles) cv2.imshow('perturbed', warped) key = cv2.waitKey() if key == ord('n'): break elif key == 27: sys.exit()
cv2.namedWindow('triangles') cv2.namedWindow('warp') smodel = ShapeModel.load('data/shape.npz') params = smodel.get_params(200, 150, 150) ref = smodel.calc_shape(params) ref = ref.reshape((len(ref) // 2, 2)) verts = get_vertices(ref) muct = MuctDataset() muct.load(clean=True) if args.use_texturemapper: from pyaam.texturemapper import TextureMapper tm = TextureMapper(480, 640) warp_triangles = tm.warp_triangles for name, tag, lmks, flipped in muct.iterdata(): img = muct.image(name) pts = lmks.reshape((len(lmks) // 2, 2)) aabb = get_aabb(pts) if args.normalize: img = normalize(img, aabb) orig = img.copy() if args.show_triangles: draw_polygons(img, pts[verts], Color.blue) cv2.imshow('triangles', img) warped = warp_triangles(orig, pts[verts], ref[verts]) if args.show_triangles: draw_polygons(warped, ref[verts], Color.blue)
def test_aam(images, landmarks, smodel, tmodel, R, ref_shape): cv2.namedWindow('original') cv2.namedWindow('fitted') cv2.namedWindow('shape') tm = TextureMapper(480, 640) tri = get_vertices(ref_shape) for i in xrange(len(landmarks)): img = next(images) cv2.imshow('original', img) lmks = landmarks[i].reshape(ref_shape.shape) # detect face pts = detector.detect(img) # get params for detected face shape s_params = smodel.calc_params(pts) # mean texture t_params = np.zeros(tmodel.num_modes()) # concatenate parameters params = np.concatenate((s_params, t_params)) shape, texture = get_instance(params, smodel, tmodel) warped = draw_face(img, shape, texture, ref_shape, tm.warp_triangles) cv2.imshow('fitted', warped) img2 = img.copy() draw_muct_shape(img2, shape.ravel()) cv2.imshow('shape', img2) key = cv2.waitKey() if key == ord(' '): pass elif key == ord('n'): continue elif key == 27: sys.exit() shape, texture = get_instance(params, smodel, tmodel) g_image = sample_texture(img, shape, ref_shape, tm.warp_triangles) # compute residual residual = g_image - texture # evaluate error E0 = np.dot(residual, residual) # predict model displacements pert = R.dot(residual) for i in xrange(MAX_ITER): shape, texture = get_instance(params, smodel, tmodel) g_image = sample_texture(img, shape, ref_shape, tm.warp_triangles) # compute residual residual = g_image - texture # predict model displacements pert = R.dot(residual) for alpha in (1.5, 1, 0.5, 0.25, 0.125): new_params = params - alpha * pert shape, texture = get_instance(new_params, smodel, tmodel) g_image = sample_texture(img, shape, ref_shape, tm.warp_triangles) residual = g_image - texture Ek = np.dot(residual, residual) if Ek < E0: params = new_params E0 = Ek break shape, texture = get_instance(params, smodel, tmodel) warped = draw_face(img, shape, texture, ref_shape, tm.warp_triangles) cv2.imshow('fitted', warped) img2 = img.copy() draw_muct_shape(img2, shape.ravel()) cv2.imshow('shape', img2) key = cv2.waitKey() if key == ord(' '): continue elif key == ord('n'): break elif key == 27: sys.exit()