def test_texturedtrimesh_copy():
points = np.ones([10, 3])
tcoords = np.ones([10, 3])
trilist = np.ones([10, 3])
landmarks = PointCloud(np.ones([3, 3]), copy=False)
landmarks_im = PointCloud(np.ones([3, 2]), copy=False)
pixels = np.ones([10, 10, 1])
texture = Image(pixels, copy=False)
texture.landmarks['test_im'] = landmarks_im
ttmesh = TexturedTriMesh(points, tcoords, texture, trilist=trilist,
copy=False)
ttmesh.landmarks['test'] = landmarks
ttmesh_copy = ttmesh.copy()
assert (not is_same_array(ttmesh_copy.points, ttmesh.points))
assert (not is_same_array(ttmesh_copy.trilist, ttmesh.trilist))
assert (not is_same_array(ttmesh_copy.tcoords.points,
ttmesh.tcoords.points))
assert (not is_same_array(ttmesh_copy.texture.pixels,
ttmesh.texture.pixels))
assert (not is_same_array(
ttmesh_copy.texture.landmarks['test_im'].lms.points,
ttmesh.texture.landmarks['test_im'].lms.points))
assert (not is_same_array(ttmesh_copy.landmarks['test'].lms.points,
ttmesh.landmarks['test'].lms.points))
def build(self, **kwargs):
if self.texture is not None:
mesh = TexturedTriMesh(self.points, self.trilist,
self.tcoords, self.texture)
else:
mesh = TriMesh(self.points, self.trilist)
if self.landmarks is not None:
mesh.landmarks.add_reference_landmarks(self.landmarks)
mesh.legacy = {'path_and_filename': self.path_and_filename}
return mesh
def _instance(self, shape_instance, texture_instance, landmark_group):
# Create trimesh
# restrict the texture to 0-1
# texture_instance.pixels = np.clip(texture_instance.pixels, 0, 1)
trimesh = TexturedTriMesh(shape_instance.points,
trilist=shape_instance.trilist,
tcoords=self.tcoords.points,
texture=texture_instance)
# Attach landmarks to trimesh
trimesh.landmarks[landmark_group] = self.landmarks
# Return trimesh
return trimesh
def _instance(self, shape_instance, texture_instance, landmark_group):
# Create trimesh
# restrict the texture to 0-1
# texture_instance.pixels = np.clip(texture_instance.pixels, 0, 1)
trimesh = TexturedTriMesh(shape_instance.points,
trilist=shape_instance.trilist,
tcoords=self.tcoords.points,
texture=texture_instance)
# Attach landmarks to trimesh
trimesh.landmarks[landmark_group] = self.landmarks
# Return trimesh
return trimesh
def test_texturedtrimesh_init_2d_grid():
tm = TexturedTriMesh.init_2d_grid([10, 10])
assert tm.n_points == 100
assert tm.n_dims == 2
# 162 = 9 * 9 * 2
assert_allclose(tm.trilist.shape, (162, 3))
assert_allclose(tm.range(), [9, 9])
def test_texturedtrimesh_init_2d_grid():
tm = TexturedTriMesh.init_2d_grid([10, 10])
assert tm.n_points == 100
assert tm.n_dims == 2
# 162 = 9 * 9 * 2
assert_allclose(tm.trilist.shape, (162, 3))
assert_allclose(tm.range(), [9, 9])
def test_textured_trimesh_viewer():
with raises(Menpo3dMissingError):
TexturedTriMesh(fake_triangle,
fake_tcoords,
fake_texture,
trilist=fake_trilist,
copy=False).view()
def test_texturedtrimesh_init_from_depth_image():
fake_z = np.random.uniform(size=(10, 10))
tm = TexturedTriMesh.init_from_depth_image(Image(fake_z))
assert tm.n_points == 100
assert tm.n_dims == 3
assert_allclose(tm.range()[:2], [9, 9])
assert tm.points[:, -1].max() <= 1.0
assert tm.points[:, -1].min() >= 0.0
def test_texturedtrimesh_init_from_depth_image():
fake_z = np.random.uniform(size=(10, 10))
tm = TexturedTriMesh.init_from_depth_image(Image(fake_z))
assert tm.n_points == 100
assert tm.n_dims == 3
assert_allclose(tm.range()[:2], [9, 9])
assert tm.points[:, -1].max() <= 1.0
assert tm.points[:, -1].min() >= 0.0
def test_texturedtrimesh_creation_copy_true():
points = np.array([[0, 0, 0], [1, 0, 0], [1, 1, 0], [0, 1, 0]])
trilist = np.array([[0, 1, 3], [1, 2, 3]])
pixels = np.ones([10, 10, 1])
tcoords = np.ones([4, 2])
texture = Image(pixels, copy=False)
ttm = TexturedTriMesh(points, tcoords, texture, trilist=trilist, copy=True)
assert (not is_same_array(ttm.points, points))
assert (not is_same_array(ttm.trilist, trilist))
assert (not is_same_array(ttm.tcoords.points, tcoords))
assert (not is_same_array(ttm.texture.pixels, pixels))
def test_texturedtrimesh_init_from_depth_image_masked():
fake_z = np.random.uniform(size=(10, 10))
mask = np.zeros(fake_z.shape, dtype=np.bool)
mask[2:6, 2:6] = True
im = MaskedImage(fake_z, mask=mask)
tm = TexturedTriMesh.init_from_depth_image(im)
assert tm.n_points == 16
assert tm.n_dims == 3
assert_allclose(tm.range()[:2], [3, 3])
assert tm.points[:, -1].max() <= 1.0
assert tm.points[:, -1].min() >= 0.0
def test_texturedtrimesh_init_from_depth_image_masked():
fake_z = np.random.uniform(size=(10, 10))
mask = np.zeros(fake_z.shape, dtype=np.bool)
mask[2:6, 2:6] = True
im = MaskedImage(fake_z, mask=mask)
tm = TexturedTriMesh.init_from_depth_image(im)
assert tm.n_points == 16
assert tm.n_dims == 3
assert_allclose(tm.range()[:2], [3, 3])
assert tm.points[:, -1].max() <= 1.0
assert tm.points[:, -1].min() >= 0.0
def _construct_shape_type(points, trilist, tcoords, texture,
colour_per_vertex):
r"""
Construct the correct Shape subclass given the inputs. TexturedTriMesh
can only be created when tcoords and texture are available. ColouredTriMesh
can only be created when colour_per_vertex is non None and TriMesh
can only be created when trilist is non None. Worst case fall back is
PointCloud.
Parameters
----------
points : ``(N, D)`` `ndarray`
The N-D points.
trilist : ``(N, 3)`` `ndarray`` or ``None``
Triangle list or None.
tcoords : ``(N, 2)`` `ndarray` or ``None``
Texture coordinates.
texture : :map:`Image` or ``None``
Texture.
colour_per_vertex : ``(N, 1)`` or ``(N, 3)`` `ndarray` or ``None``
The colour per vertex.
Returns
-------
shape : :map:`PointCloud` or subclass
The correct shape for the given inputs.
"""
# Four different outcomes - either we have a textured mesh, a coloured
# mesh or just a plain mesh or we fall back to a plain pointcloud.
if trilist is None:
obj = PointCloud(points, copy=False)
elif tcoords is not None and texture is not None:
obj = TexturedTriMesh(points,
tcoords,
texture,
trilist=trilist,
copy=False)
elif colour_per_vertex is not None:
obj = ColouredTriMesh(points,
trilist=trilist,
colours=colour_per_vertex,
copy=False)
else:
# TriMesh fall through
obj = TriMesh(points, trilist=trilist, copy=False)
if tcoords is not None and texture is None:
warnings.warn('tcoords were found, but no texture was recovered, '
'reverting to an untextured mesh.')
if texture is not None and tcoords is None:
warnings.warn('texture was found, but no tcoords were recovered, '
'reverting to an untextured mesh.')
return obj
def upsample_eos_low_res_to_fw(eos_mesh_low_res):
bc_fw_on_eos_low_res, tri_index_fw_on_eos_low_res = load_fw_on_eos_low_res_settings(
)
effective_fw_pc = eos_mesh_low_res.project_barycentric_coordinates(
bc_fw_on_eos_low_res, tri_index_fw_on_eos_low_res)
tcoords = eos_mesh_low_res.barycentric_coordinate_interpolation(
eos_mesh_low_res.tcoords.points, bc_fw_on_eos_low_res,
tri_index_fw_on_eos_low_res)
effective_fw = TexturedTriMesh(effective_fw_pc.points,
tcoords,
eos_mesh_low_res.texture,
trilist=load_basel_kf_trilist())
return effective_fw
def fit_model():
# # move all user related image files in own folder
# newdir = os.path.join(os.getcwd(), "User_Image")
# delete_folder_contents(newdir)
# newfile = os.path.join(newdir, "original_image.jpg")
# os.rename(os.getcwd(), newfile)
# import image
image = mio.import_images("User_Image/*.jpg")[0]
# the morphable model
mm = ColouredMorphableModel(shape_model, texture_model, landmarks,
holistic_features=menpo.feature.fast_dsift, diagonal=185)
# bb: bounding_box
bb = detect(image)[0]
fitter = LucasKanadeMMFitter(mm, n_shape=200, n_texture=200, n_samples=8000, n_scales=1)
# initial_shape: An image of points indicating landmarks on the face
initial_shape = aam_fitter.fit_from_bb(image, bb).final_shape
result = fitter.fit_from_shape(image, initial_shape, max_iters=30,
camera_update=True,
focal_length_update=False,
reconstruction_weight=1,
shape_prior_weight=1e7,
texture_prior_weight=1.,
landmarks_prior_weight=1e5,
return_costs=False,
init_shape_params_from_lms=False)
# The mesh in image coordinates
mesh_in_img = transform(result, result.final_mesh)
# A flattened image showing the face extracted from the orginal image
uv = extract_texture(mesh_in_img, image)
textured_mesh = TexturedTriMesh(result.final_mesh.points,
tcoords=image_coords_to_tcoords(uv.shape).apply(tcoords).points,
texture=uv,
trilist=result.final_mesh.trilist)
# return textured_mesh
m3io.export_textured_mesh(textured_mesh, 'User_Image/texture.obj', extension='obj', overwrite=True)
def fit(imagepath):
image = mio.import_image(imagepath, normalize=False)
if len(image.pixels.shape) == 2:
image.pixels = np.stack([image.pixels, image.pixels, image.pixels])
if image.pixels.shape[0] == 1:
image.pixels = np.concatenate(
[image.pixels, image.pixels, image.pixels], axis=0)
print(image.pixels_with_channels_at_back().shape)
bb = detect(image.pixels_with_channels_at_back())[0]
initial_shape = aam_fitter.fit_from_bb(image, bb).final_shape
result = fitter.fit_from_shape(image,
initial_shape,
max_iters=40,
camera_update=True,
focal_length_update=False,
reconstruction_weight=1,
shape_prior_weight=.4e8,
texture_prior_weight=1.,
landmarks_prior_weight=1e5,
return_costs=True,
init_shape_params_from_lms=False)
mesh = ColouredTriMesh(result.final_mesh.points, result.final_mesh.trilist)
def transform(mesh):
return result._affine_transforms[-1].apply(
result.camera_transforms[-1].apply(mesh))
mesh_in_img = transform(lambertian_shading(mesh))
expr_dir = image.path.parent
p = image.path.stem
raster = rasterize_mesh(mesh_in_img, image.shape)
uv_shape = (600, 1000)
template = shape_model.mean()
unwrapped_template = optimal_cylindrical_unwrap(template).apply(template)
minimum = unwrapped_template.bounds(boundary=0)[0]
unwrapped_template = Translation(-minimum).apply(unwrapped_template)
unwrapped_template.points = unwrapped_template.points[:, [1, 0]]
unwrapped_template.points[:, 0] = unwrapped_template.points[:, 0].max(
) - unwrapped_template.points[:, 0]
unwrapped_template.points *= np.array([.40, .31])
unwrapped_template.points *= np.array([uv_shape])
bcoords_img, tri_index_img = rasterize_barycentric_coordinate_images(
unwrapped_template, uv_shape)
TI = tri_index_img.as_vector()
BC = bcoords_img.as_vector(keep_channels=True).T
def masked_texture(mesh_in_image, background):
sample_points_3d = mesh_in_image.project_barycentric_coordinates(
BC, TI)
texture = bcoords_img.from_vector(
background.sample(sample_points_3d.points[:, :2]))
return texture
uv = masked_texture(mesh_in_img, image)
t = TexturedTriMesh(
result.final_mesh.points,
image_coords_to_tcoords(uv.shape).apply(unwrapped_template).points, uv,
mesh_in_img.trilist)
m3io.export_textured_mesh(t,
str(expr_dir / Path(p).with_suffix('.mesh.obj')),
overwrite=True)
mio.export_image(raster,
str(expr_dir / Path(p).with_suffix('.render.jpg')),
overwrite=True)
def textured_trimesh_viewer_test():
TexturedTriMesh(fake_triangle,
fake_tcoords,
fake_texture,
trilist=fake_trilist,
copy=False).view()
def generate_interpolation_images(run_id,
snapshot=None,
grid_size=[1, 1],
image_shrink=1,
image_zoom=1,
duration_sec=60.0,
smoothing_sec=1.0,
mp4=None,
mp4_fps=30,
mp4_codec='libx265',
mp4_bitrate='16M',
random_seed=1000,
minibatch_size=8):
network_pkl = misc.locate_network_pkl(run_id, snapshot)
if mp4 is None:
mp4 = misc.get_id_string_for_network_pkl(network_pkl) + '-lerp.mp4'
num_frames = int(np.rint(duration_sec * mp4_fps))
random_state = np.random.RandomState(random_seed)
print('Loading network from "%s"...' % network_pkl)
G, D, Gs = misc.load_network_pkl(run_id, snapshot)
print('Generating latent vectors...')
shape = [num_frames, np.prod(grid_size)
] + Gs.input_shape[1:] # [frame, image, channel, component]
all_latents = random_state.randn(*shape).astype(np.float32)
all_latents = scipy.ndimage.gaussian_filter(
all_latents, [smoothing_sec * mp4_fps] + [0] * len(Gs.input_shape),
mode='wrap')
all_latents /= np.sqrt(np.mean(np.square(all_latents)))
lsfm_model = m3io.import_lsfm_model(
'/home/baris/Projects/faceganhd/models/all_all_all.mat')
lsfm_tcoords = \
mio.import_pickle('/home/baris/Projects/team members/stelios/UV_spaces_V2/UV_dicts/full_face/512_UV_dict.pkl')[
'tcoords']
lsfm_params = []
result_subdir = misc.create_result_subdir(config.result_dir, config.desc)
for png_idx in range(int(num_frames / minibatch_size)):
start = time.time()
print('Generating png %d-%d / %d... in ' %
(png_idx * minibatch_size,
(png_idx + 1) * minibatch_size, num_frames),
end='')
latents = all_latents[png_idx * minibatch_size:(png_idx + 1) *
minibatch_size, 0]
labels = np.zeros([latents.shape[0], 0], np.float32)
images = Gs.run(latents,
labels,
minibatch_size=minibatch_size,
num_gpus=config.num_gpus,
out_shrink=image_shrink)
for i in range(minibatch_size):
texture = Image(np.clip(images[i, 0:3] / 2 + 0.5, 0, 1))
mesh_raw = from_UV_2_3D(Image(images[i, 3:6]))
normals = images[i, 6:9]
normals_norm = (normals - normals.min()) / (normals.max() -
normals.min())
mesh = lsfm_model.reconstruct(mesh_raw)
lsfm_params.append(lsfm_model.project(mesh_raw))
t_mesh = TexturedTriMesh(mesh.points, lsfm_tcoords.points, texture,
mesh.trilist)
m3io.export_textured_mesh(
t_mesh,
os.path.join(result_subdir,
'%06d.obj' % (png_idx * minibatch_size + i)),
texture_extension='.png')
mio.export_image(
Image(normals_norm),
os.path.join(result_subdir,
'%06d_nor.png' % (png_idx * minibatch_size + i)))
print('%0.2f seconds' % (time.time() - start))
mio.export_pickle(lsfm_params,
os.path.join(result_subdir, 'lsfm_params.pkl'))
open(os.path.join(result_subdir, '_done.txt'), 'wt').close()
def generate_fake_images(run_id,
snapshot=None,
grid_size=[1, 1],
batch_size=8,
num_pngs=1,
image_shrink=1,
png_prefix=None,
random_seed=1000,
minibatch_size=8):
network_pkl = misc.locate_network_pkl(run_id, snapshot)
if png_prefix is None:
png_prefix = misc.get_id_string_for_network_pkl(network_pkl) + '-'
random_state = np.random.RandomState(random_seed)
print('Loading network from "%s"...' % network_pkl)
G, D, Gs = misc.load_network_pkl(run_id, snapshot)
lsfm_model = m3io.import_lsfm_model(
'/home/baris/Projects/faceganhd/models/all_all_all.mat')
lsfm_tcoords = \
mio.import_pickle('/home/baris/Projects/team members/stelios/UV_spaces_V2/UV_dicts/full_face/512_UV_dict.pkl')[
'tcoords']
lsfm_params = []
result_subdir = misc.create_result_subdir(config.result_dir, config.desc)
for png_idx in range(int(num_pngs / batch_size)):
start = time.time()
print('Generating png %d-%d / %d... in ' %
(png_idx * batch_size, (png_idx + 1) * batch_size, num_pngs),
end='')
latents = misc.random_latents(np.prod(grid_size) * batch_size,
Gs,
random_state=random_state)
labels = np.zeros([latents.shape[0], 0], np.float32)
images = Gs.run(latents,
labels,
minibatch_size=minibatch_size,
num_gpus=config.num_gpus,
out_shrink=image_shrink)
for i in range(batch_size):
if images.shape[1] == 3:
mio.export_pickle(
images[i],
os.path.join(
result_subdir,
'%s%06d.pkl' % (png_prefix, png_idx * batch_size + i)))
# misc.save_image(images[i], os.path.join(result_subdir, '%s%06d.png' % (png_prefix, png_idx*batch_size+i)), [0,255], grid_size)
elif images.shape[1] == 6:
mio.export_pickle(images[i][3:6],
os.path.join(
result_subdir, '%s%06d.pkl' %
(png_prefix, png_idx * batch_size + i)),
overwrite=True)
misc.save_image(
images[i][0:3],
os.path.join(
result_subdir,
'%s%06d.png' % (png_prefix, png_idx * batch_size + i)),
[-1, 1], grid_size)
elif images.shape[1] == 9:
texture = Image(np.clip(images[i, 0:3] / 2 + 0.5, 0, 1))
mesh_raw = from_UV_2_3D(Image(images[i, 3:6]))
normals = images[i, 6:9]
normals_norm = (normals - normals.min()) / (normals.max() -
normals.min())
mesh = lsfm_model.reconstruct(mesh_raw)
lsfm_params.append(lsfm_model.project(mesh_raw))
t_mesh = TexturedTriMesh(mesh.points, lsfm_tcoords.points,
texture, mesh.trilist)
m3io.export_textured_mesh(
t_mesh,
os.path.join(result_subdir,
'%06d.obj' % (png_idx * minibatch_size + i)),
texture_extension='.png')
mio.export_image(
Image(normals_norm),
os.path.join(
result_subdir,
'%06d_nor.png' % (png_idx * minibatch_size + i)))
shape = images[i, 3:6]
shape_norm = (shape - shape.min()) / (shape.max() -
shape.min())
mio.export_image(
Image(shape_norm),
os.path.join(
result_subdir,
'%06d_shp.png' % (png_idx * minibatch_size + i)))
mio.export_pickle(
t_mesh,
os.path.join(result_subdir,
'%06d.pkl' % (png_idx * minibatch_size + i)))
print('%0.2f seconds' % (time.time() - start))
open(os.path.join(result_subdir, '_done.txt'), 'wt').close()
def generate_interpolation_images(run_id,
snapshot=None,
grid_size=[1, 1],
image_shrink=1,
image_zoom=1,
duration_sec=60.0,
smoothing_sec=1.0,
mp4=None,
mp4_fps=30,
mp4_codec='libx265',
mp4_bitrate='16M',
random_seed=1000,
minibatch_size=8):
network_pkl = misc.locate_network_pkl(run_id, snapshot)
if mp4 is None:
mp4 = misc.get_id_string_for_network_pkl(network_pkl) + '-lerp.mp4'
num_frames = int(np.rint(duration_sec * mp4_fps))
random_state = np.random.RandomState(random_seed)
print('Loading network from "%s"...' % network_pkl)
G, D, Gs = misc.load_network_pkl(run_id, snapshot)
print('Generating latent vectors...')
shape = [num_frames, np.prod(grid_size)] + [
Gs.input_shape[1:][0] + Gs.input_shapes[1][1:][0]
] # [frame, image, channel, component]
all_latents = random_state.randn(*shape).astype(np.float32)
all_latents = scipy.ndimage.gaussian_filter(
all_latents, [smoothing_sec * mp4_fps] + [0] * len(Gs.input_shape),
mode='wrap')
all_latents /= np.sqrt(np.mean(np.square(all_latents)))
#10 10 10 10 5 3 10
# model = mio.import_pickle('../models/lsfm_shape_model_fw.pkl')
# facesoft_model = mio.import_pickle('../models/facesoft_id_and_exp_3d_face_model.pkl')['shape_model']
# lsfm_model = m3io.import_lsfm_model('/home/baris/Projects/faceganhd/models/all_all_all.mat')
# model_mean = lsfm_model.mean().copy()
# mask = mio.import_pickle('../UV_spaces_V2/mask_full_2_crop.pkl')
lsfm_tcoords = \
mio.import_pickle('512_UV_dict.pkl')['tcoords']
lsfm_params = []
result_subdir = misc.create_result_subdir(config_test.result_dir,
config_test.desc)
for png_idx in range(int(num_frames / minibatch_size)):
start = time.time()
print('Generating png %d-%d / %d... in ' %
(png_idx * minibatch_size,
(png_idx + 1) * minibatch_size, num_frames),
end='')
latents = all_latents[png_idx * minibatch_size:(png_idx + 1) *
minibatch_size, 0, :Gs.input_shape[1:][0]]
labels = all_latents[png_idx * minibatch_size:(png_idx + 1) *
minibatch_size, 0, Gs.input_shape[1:][0]:]
labels_softmax = softmax(labels) * np.array([10, 10, 10, 10, 5, 3, 10])
images = Gs.run(latents,
labels_softmax,
minibatch_size=minibatch_size,
num_gpus=config_test.num_gpus,
out_shrink=image_shrink)
for i in range(minibatch_size):
texture = Image(np.clip(images[i, 0:3] / 2 + 0.5, 0, 1))
img_shape = ndimage.gaussian_filter(images[i, 3:6],
sigma=(0, 3, 3),
order=0)
mesh_raw = from_UV_2_3D(Image(img_shape),
topology='full',
uv_layout='oval')
# model_mean.points[mask,:] = mesh_raw.points
normals = images[i, 6:9]
normals_norm = (normals - normals.min()) / (normals.max() -
normals.min())
mesh = mesh_raw #facesoft_model.reconstruct(model_mean).from_mask(mask)
# lsfm_params.append(lsfm_model.project(mesh_raw))
t_mesh = TexturedTriMesh(mesh.points, lsfm_tcoords.points, texture,
mesh.trilist)
m3io.export_textured_mesh(
t_mesh,
os.path.join(result_subdir,
'%06d.obj' % (png_idx * minibatch_size + i)),
texture_extension='.png')
fix_obj(
os.path.join(result_subdir,
'%06d.obj' % (png_idx * minibatch_size + i)))
mio.export_image(
Image(normals_norm),
os.path.join(result_subdir,
'%06d_nor.png' % (png_idx * minibatch_size + i)))
print('%0.2f seconds' % (time.time() - start))
mio.export_pickle(lsfm_params,
os.path.join(result_subdir, 'lsfm_params.pkl'))
open(os.path.join(result_subdir, '_done.txt'), 'wt').close()