def create_feature_space(feature_matrix, example_image, feature_space_name):
feature_space_images = []
N = feature_matrix.shape[0]
for i, n in enumerate(feature_matrix):
new_im = MaskedNDImage.blank(example_image.shape, mask=example_image.mask, n_channels=n.shape[1])
new_im.from_vector_inplace(n.flatten())
new_im.landmarks = example_image.landmarks
feature_space_images.append(new_im)
print_replace_line('Image {0} of {1}'.format(i + 1, N))
cPickle.dump(images, open('/vol/atlas/pts08/cvpr/frgc_spring2003_4_{0}.pkl'.format(feature_space_name), 'wb'), protocol=2)
return feature_space_images
def test_reconstruction(im):
from copy import deepcopy
from pybug.image import MaskedNDImage, DepthImage
im = deepcopy(im)
new_im = MaskedNDImage.blank(im.shape, mask=im.mask, n_channels=3)
im.rebuild_mesh()
n = im.mesh.vertex_normals
new_im.from_vector_inplace(n.flatten())
g = gradient_field_from_normals(new_im)
d = frankotchellappa(g.pixels[..., 0], g.pixels[..., 1])
im.view(mode='mesh', normals=n, mask_points=20)
DepthImage(d - np.min(d)).view_new(mode='mesh')
def test_reconstruction(im):
from copy import deepcopy
from pybug.image import MaskedNDImage, DepthImage
im = deepcopy(im)
new_im = MaskedNDImage.blank(im.shape, mask=im.mask, n_channels=3)
im.rebuild_mesh()
n = im.mesh.vertex_normals
new_im.from_vector_inplace(n.flatten())
g = gradient_field_from_normals(new_im)
d = frankotchellappa(g.pixels[..., 0], g.pixels[..., 1])
im.view(mode='mesh', normals=n, mask_points=20)
DepthImage(d - np.min(d)).view_new(mode='mesh')
def create_feature_space(feature_matrix, example_image, feature_space_name):
feature_space_images = []
N = feature_matrix.shape[0]
for i, n in enumerate(feature_matrix):
new_im = MaskedNDImage.blank(example_image.shape,
mask=example_image.mask,
n_channels=n.shape[1])
new_im.from_vector_inplace(n.flatten())
new_im.landmarks = example_image.landmarks
feature_space_images.append(new_im)
print_replace_line('Image {0} of {1}'.format(i + 1, N))
cPickle.dump(images,
open(
'/vol/atlas/pts08/cvpr/frgc_spring2003_4_{0}.pkl'.format(
feature_space_name), 'wb'),
protocol=2)
return feature_space_images
try:
intrinsic_mean_normals = model["intrinsic_mean_normals"]
except Exception:
pass
# <codecell>
from pybug.image import MaskedNDImage
from pybug.io import auto_import
from pybug.landmark import labeller, ibug_68_closed_mouth
from landmarks import ibug_68_edge
sfs_index = 2
bej = auto_import("/vol/atlas/databases/alex_images/bej*.ppm")
# Create a 4 channel image where each channel is the greyscale of an image
ground_truth_images = MaskedNDImage(np.concatenate([im.as_greyscale().pixels for im in bej], axis=2))
intensity_image = bej[sfs_index].as_greyscale()
intensity_image.landmarks = bej[0].landmarks
ground_truth_images.landmarks["PTS"] = bej[0].landmarks["PTS"]
labeller([ground_truth_images, intensity_image], "PTS", ibug_68_closed_mouth)
# labeller([ground_truth_images, intensity_image], 'PTS', ibug_68_edge)
lights = np.array([[0.5, 0.4, 2], [-0.5, 0.4, 2], [-0.5, -0.4, 2], [0.5, -0.4, 2]])
# <codecell>
from pybug.transform.tps import TPS
from warp import build_similarity_transform
std_depth_error_results = np.zeros([len(photoface_subjects), len(feature_spaces)])
std_angular_error_results = np.zeros([len(photoface_subjects), len(feature_spaces)])
# 5 feature spaces + ground truth
normals = dict(zip(photoface_subjects, [{}, {}, {}, {}, {}, {}, {}]))
for s in normals.values():
s.update(zip(['ground_truth'] + feature_spaces, [None] * (len(feature_spaces) + 1)))
for i, subject_id in enumerate(photoface_subjects):
print "Running experiment for {0}".format(subject_id)
subject_images = auto_import(
'/vol/atlas/databases/alex_images/{0}*.ppm'.format(subject_id))
# Create a 4 channel image where each channel is the greyscale of an image
ground_truth_images = MaskedNDImage(
np.concatenate([im.as_greyscale().pixels
for im in subject_images], axis=2))
# Choose the third image as the reconstruction candidate
intensity_image = subject_images[sfs_index].as_greyscale()
# The first image is the only landmarked one
intensity_image.landmarks = subject_images[0].landmarks
# Pass landmarks to all ground truth images
ground_truth_images.landmarks['PTS'] = intensity_image.landmarks['PTS']
# Label with correct labels
labeller([ground_truth_images, intensity_image],
'PTS', ibug_68_closed_mouth)
# Constrain to mask
normal_model = model['appearance_model']
reference_frame = model['template']
mean_normals = model['mean_normals']
reference_frame = model['template']
try:
intrinsic_mean_normals = model['intrinsic_mean_normals']
except Exception:
intrinsic_mean_normals = None
# Estimate light direction for image
I = intensity_image.as_vector()
estimate_light = np.dot(pinv2(mean_normals), I)
print estimate_light
# Perform SFS
warped_intensity_image = MaskedNDImage(intensity_image.pixels.copy(),
mask=intensity_image.mask)
initial_estimate_image = warped_intensity_image.from_vector(
mean_normals.copy(), n_channels=3)
mapping_object = build_mapping_object(feature_space,
initial_estimate_image,
intrinsic_mean_normals)
# Normalise the image so that it has unit albedo?
#warped_intensity_image.masked_pixels /= ground_truth_albedo.masked_pixels
#warped_intensity_image.masked_pixels /= np.max(warped_intensity_image.masked_pixels)
reconstructed_normals = sfs(warped_intensity_image,
initial_estimate_image, normal_model,
estimate_light, n_iters=200,
mapping_object=mapping_object)
normals[subject_id][feature_space] = reconstructed_normals
from pybug.io import auto_import
from pybug.image import RGBImage, MaskedNDImage
images = auto_import('/Users/joan/PhD/DataBases/lfpw/trainset/*.png')
images = [i.as_greyscale() if type(i) is RGBImage else i for i in images]
new_images = []
for i in images:
img = MaskedNDImage(i.pixels)
img.landmarks = i.landmarks
new_images.append(img)
images = new_images
del new_images
# <codecell>
from pybug.activeappearancemodel import aam_builder, AAM
from pybug.transform.tps import TPS
from pybug.landmark import ibug_68_trimesh
options = {'triangulation': {'function': ibug_68_trimesh, 'label': 'ibug_68_trimesh'},
'features': {'type': None, 'options': {'kwargs': None}},
'max_shape_components': 25,
'max_appearance_components': 250}
aam = aam_builder(images, **options)
import numpy as np
# <codecell>
return feature_space_images # <markdowncell> # ## Generate the frame of reference via a Similarity Transform # <codecell> from landmarks import ibug_68_edge # Pull out the landmarks labeller(images, 'PTS', ibug_68_edge) shapes = [img.landmarks['ibug_68_edge'].lms for img in images] # <codecell> ref_frame = MaskedNDImage.blank([480, 360]) # <codecell> from warp import build_similarity_transform # Warp each of the images to the reference image sim_transforms = [build_similarity_transform(shape) for shape in shapes] warped_images = [img.warp_to(ref_frame.mask, t) for img, t in zip(images, sim_transforms)] # <markdowncell> # ## Calculate the normal matrix for all the images # <codecell> normal_matrix = extract_normals(warped_images)
def aam_builder(path, max_images=None, group='PTS', label='all',
crop_boundary=0.2, interpolator='scipy',
scale=1, max_shape_components=25, reference_frame_boundary=3,
labels=[], triangulation_func=ibug_68_trimesh,
triangulation_label='ibug_68_trimesh',
n_multiresolution_levels=3,
transform_cls=PiecewiseAffineTransform,
max_appearance_components=250):
# TODO:
print '- Loading images from:' + path
# load images
images = auto_import(path, max_images=max_images)
# convert to greyscale
images = [i.as_greyscale() if type(i) is RGBImage else i for i in images]
# crop images around their landmarks
for i in images:
i.crop_to_landmarks_proportion(crop_boundary, group=group, label=label)
# TODO:
print '- Normalizing object scales'
# extract shapes
shapes = [i.landmarks[group][label].lms for i in images]
# define reference shape
reference_shape = PointCloud(np.mean([s.points for s in shapes], axis=0))
# compute scale difference between all shapes and reference shape
scales = [UniformScale.align(s, reference_shape).as_vector()
for s in shapes]
# rescale all images using previous scales
images = [i.rescale(s, interpolator=interpolator)
for i, s in zip(images, scales)]
# extract rescaled shapes
shapes = [i.landmarks[group][label].lms for i in images]
# TODO:
pyramid_iterator = [pyramid_gaussian(i.pixels) for i in images]
# TODO:
print '- Building shape model'
# centralize shapes
centered_shapes = [Translation(-s.centre).apply(s) for s in shapes]
# align centralized shape using Procrustes Analysis
gpa = GeneralizedProcrustesAnalysis(centered_shapes)
aligned_shapes = [s.aligned_source for s in gpa.transforms]
# TODO:
# scale shape if necessary
if scale is not 1:
aligned_shapes = [Scale(scale, n_dims=reference_shape.n_dims).apply(s)
for s in aligned_shapes]
# build shape model
shape_model = PCAModel(aligned_shapes)
# trim shape model if required
if max_shape_components is not None:
shape_model.trim_components(max_shape_components)
# TODO:
print '- Building reference frame'
# scale reference shape if necessary
if scale is not 1:
reference_shape = Scale(
scale, n_dims=reference_shape.n_dims).apply(reference_shape)
# compute lower bound
lower_bound = reference_shape.bounds(
boundary=reference_frame_boundary)[0]
# translate reference shape using lower bound
reference_landmarks = Translation(-lower_bound).apply(
reference_shape)
# compute reference frame resolution
reference_resolution = reference_landmarks.range(
boundary=reference_frame_boundary)
# build reference frame
reference_frame = MaskedNDImage.blank(reference_resolution)
# assign landmarks using the default group
reference_frame.landmarks[group] = reference_landmarks
# label reference frame
for l in labels:
labeller([reference_frame], group, l)
# check for precomputed triangulation
if triangulation_func is not None:
labeller([reference_frame], group, triangulation_func)
trilist = reference_frame.landmarks[triangulation_label].lms.trilist
else:
trilist = None
# mask reference frame
reference_frame.constrain_mask_to_landmarks(group=group, trilist=trilist)
# TODO:
print '- Building Apperance Models'
# extract landmarks
landmarks = [i.landmarks[group][label].lms for i in images]
# initialize list of appearance models
appearance_model_list = []
# for each level
for j in range(0, n_multiresolution_levels):
print ' - Level {}'.format(j)
# obtain images
level_images = [MaskedNDImage(p.next()) for p in
pyramid_iterator]
# rescale and reassign landmarks if necessary
for li, i in zip(level_images, images):
li.landmarks[group] = i.landmarks[group]
li.landmarks[group].lms.points = (i.landmarks[group].lms.points /
(2 ** j))
# mask level_images
for i in level_images:
i.constrain_mask_to_landmarks(group=group, trilist=trilist)
# compute features
for i in level_images:
i.normalize_inplace()
feature_images = level_images
# compute transforms
transforms = [transform_cls(reference_frame.landmarks[group].lms,
i.landmarks[group].lms)
for i in feature_images]
# warp images
warped_images = [i.warp_to(reference_frame.mask, t,
warp_landmarks=False,
interpolator=interpolator)
for i, t in zip(feature_images, transforms)]
# assign reference landmarks using the default group
for i in warped_images:
i.landmarks[group] = reference_landmarks
# label reference frame
for l in labels:
labeller([reference_frame], group, l)
# check for precomputed triangulation
if triangulation_func is not None:
labeller([reference_frame], group, triangulation_func)
# mask warped images
for i in warped_images:
i.constrain_mask_to_landmarks(group=group, trilist=trilist)
# build appearance model
appearance_model = PCAModel(warped_images)
# trim apperance model if required
if max_appearance_components is not None:
appearance_model.trim_components(max_appearance_components)
# add appearance model to the list
appearance_model_list.append(appearance_model)
return shape_model, reference_frame, appearance_model_list
# <markdowncell>
# ## Generate the frame of reference via a Similarity Transform
# <codecell>
from landmarks import ibug_68_edge
# Pull out the landmarks
labeller(images, 'PTS', ibug_68_edge)
shapes = [img.landmarks['ibug_68_edge'].lms for img in images]
# <codecell>
ref_frame = MaskedNDImage.blank([480, 360])
# <codecell>
from warp import build_similarity_transform
# Warp each of the images to the reference image
sim_transforms = [build_similarity_transform(shape) for shape in shapes]
warped_images = [
img.warp_to(ref_frame.mask, t) for img, t in zip(images, sim_transforms)
]
# <markdowncell>
# ## Calculate the normal matrix for all the images
# <codecell>
try:
intrinsic_mean_normals = model['intrinsic_mean_normals']
except Exception:
pass
# <codecell>
from pybug.image import MaskedNDImage
from pybug.io import auto_import
from pybug.landmark import labeller, ibug_68_closed_mouth
from landmarks import ibug_68_edge
sfs_index = 2
bej = auto_import('/vol/atlas/databases/alex_images/bej*.ppm')
# Create a 4 channel image where each channel is the greyscale of an image
ground_truth_images = MaskedNDImage(
np.concatenate([im.as_greyscale().pixels for im in bej], axis=2))
intensity_image = bej[sfs_index].as_greyscale()
intensity_image.landmarks = bej[0].landmarks
ground_truth_images.landmarks['PTS'] = bej[0].landmarks['PTS']
labeller([ground_truth_images, intensity_image], 'PTS', ibug_68_closed_mouth)
# labeller([ground_truth_images, intensity_image], 'PTS', ibug_68_edge)
lights = np.array([[0.5, 0.4, 2], [-0.5, 0.4, 2], [-0.5, -0.4, 2],
[0.5, -0.4, 2]])
# <codecell>
from pybug.transform.tps import TPS
from warp import build_similarity_transform
std_angular_error_results = np.zeros([len(yaleb_subjects),
len(feature_spaces)])
# 5 feature spaces + ground truth
normals = dict(zip(yaleb_subjects, [{}, {}, {}, {}, {}, {}, {}, {}, {}, {}]))
for s in normals.values():
s.update(zip(['ground_truth'] + feature_spaces, [None] * (len(feature_spaces) + 1)))
for i, subject_id in enumerate(yaleb_subjects):
print "Running experiment for {0}".format(subject_id)
subject_images = [auto_import(os.path.join(yaleb_path, subject_id[-3:], light[0].format(subject_id)))[0]
for light in image_light_paths]
# Create a 4 channel image where each channel is the greyscale of an image
ground_truth_images = MaskedNDImage(
np.concatenate([im.pixels
for im in subject_images], axis=2))
# Choose the first image as the reconstruction candidate
# (frontal illumination)
intensity_image = deepcopy(subject_images[0])
# Pass landmarks to all ground truth images
ground_truth_images.landmarks['PTS'] = intensity_image.landmarks['PTS']
# Label with correct labels
labeller([ground_truth_images, intensity_image],
'PTS', ibug_68_closed_mouth)
# Constrain to mask
ground_truth_images.constrain_mask_to_landmarks(