def load_image_test(path, reference_shape, frame_num):
file_name = path[:-1] + "/%06d.jpg" % (frame_num)
im = mio.import_image(file_name)
im.landmarks['PTS'] = mio.import_landmark_file(path[:-1] +
"/annot/%06d.pts" %
(frame_num))
# im.landmarks['PTS'] = mio.import_landmark_file(path[:-1] + "/%06d.pts" % (frame_num))
bb_path = path[:-1] + "/bbs/%06d.pts" % (frame_num)
im.landmarks['bb'] = mio.import_landmark_file(bb_path)
im = im.crop_to_landmarks_proportion(0.3, group='bb')
reference_shape = PointCloud(reference_shape)
bb = im.landmarks['bb'].lms.bounding_box()
im.landmarks['__initial'] = align_shape_with_bounding_box(
reference_shape, bb)
im = im.rescale_to_pointcloud(reference_shape, group='__initial')
lms = im.landmarks['PTS'].lms
initial = im.landmarks['__initial'].lms
# if the image is greyscale then convert to rgb.
pixels = grey_to_rgb(im).pixels.transpose(1, 2, 0)
gt_truth = lms.points.astype(np.float32)
estimate = initial.points.astype(np.float32)
return 1, pixels.astype(np.float32).copy(), gt_truth, estimate
def process_frame(frame_name, clip, img_type, svm_p, loop=False):
"""
Applies the AAM fitter (global var) in a frame. Additionally, it might apply an
SVM to verify it's a face if required.
:param frame_name: str: Name of the frame along with extension, e.g. '000001.png'.
:param clip: str: Name of the clip.
:param img_type: str: Suffix (extension) of the frames, e.g. '.png'.
:param svm_p: dict: Required params for SVM classification.
:param loop: bool: (optional) Declares whether this is a 2nd fit for AAM (loop).
:return:
"""
global fitter
name = frame_name[:frame_name.rfind('.')]
p0 = clip.path_read_ln[0] + name + '_0.pts'
# find if this is 2nd fit or 1st.
if loop: # if 2nd fit, then if landmark is 'approved', return. Otherwise proceed.
try:
ln = import_landmark_file(p0)
copy2(p0, clip.path_write_ln[0] + name + '_0.pts')
return # if the landmark already exists, return (for performance improvement)
except ValueError:
pass
try:
ln = import_landmark_file(clip.path_read_ln[1] + name + '_0.pts')
except ValueError: # either not found or no suitable importer
return
else:
try:
ln = import_landmark_file(p0)
except ValueError: # either not found or no suitable importer
return
im = im_read_greyscale(frame_name, clip.path_frames, img_type)
if not im:
return
im.landmarks['PTS2'] = ln
# fitting can be faster if the image is cropped in advance, though
# you should save the transform to get back to the original shape,
# hence here we just leave the original image.
fr = fitter.fit_from_shape(im, im.landmarks['PTS2'].lms)
p_wr = clip.path_write_ln[0] + im.path.stem + '_0.pts'
im.landmarks['ps_pbaam'] = fr.final_shape
export_landmark_file(im.landmarks['ps_pbaam'], p_wr, overwrite=True)
# apply SVM classifier by extracting patches (is face or not).
if not svm_p['apply']:
return
im_cp = im.crop_to_landmarks_proportion(0.2, group='ps_pbaam')
im_cp = svm_p['feat'](im_cp)
im2 = warp_image_to_reference_shape(im_cp, svm_p['refFrame'], 'ps_pbaam')
_p_nd = im2.extract_patches_around_landmarks(
group='source', as_single_array=True,
patch_shape=svm_p['patch_s']).flatten()
if svm_p['clf'].decision_function(_p_nd) > 0:
copy2(p_wr, clip.path_write_ln[1] + im.path.stem + '_0.pts')
def process_frame(frame_name, clip, img_type, svm_p, loop=False):
"""
Applies the AAM fitter (global var) in a frame. Additionally, it might apply an
SVM to verify it's a face if required.
:param frame_name: str: Name of the frame along with extension, e.g. '000001.png'.
:param clip: str: Name of the clip.
:param img_type: str: Suffix (extension) of the frames, e.g. '.png'.
:param svm_p: dict: Required params for SVM classification.
:param loop: bool: (optional) Declares whether this is a 2nd fit for AAM (loop).
:return:
"""
global fitter
name = frame_name[:frame_name.rfind('.')]
p0 = clip.path_read_ln[0] + name + '_0.pts'
# find if this is 2nd fit or 1st.
if loop: # if 2nd fit, then if landmark is 'approved', return. Otherwise proceed.
try:
ln = import_landmark_file(p0)
copy2(p0, clip.path_write_ln[0] + name + '_0.pts')
return # if the landmark already exists, return (for performance improvement)
except ValueError:
pass
try:
ln = import_landmark_file(clip.path_read_ln[1] + name + '_0.pts')
except ValueError: # either not found or no suitable importer
return
else:
try:
ln = import_landmark_file(p0)
except ValueError: # either not found or no suitable importer
return
im = im_read_greyscale(frame_name, clip.path_frames, img_type)
if not im:
return
im.landmarks['PTS2'] = ln
fr = fitter.fit_from_shape(im, im.landmarks['PTS2'].lms, crop_image=0.3)
p_wr = clip.path_write_ln[0] + im.path.stem + '_0.pts'
export_landmark_file(fr.fitted_image.landmarks['final'], p_wr, overwrite=True)
# apply SVM classifier by extracting patches (is face or not).
if not svm_p['apply']:
return
im.landmarks.clear() # temp solution
im.landmarks['ps_pbaam'] = fr.fitted_image.landmarks['final']
im_cp = im.crop_to_landmarks_proportion(0.2, group='ps_pbaam')
im_cp = svm_p['feat'](im_cp)
im2 = warp_image_to_reference_shape(im_cp, svm_p['refFrame'], 'ps_pbaam')
_p_nd = im2.extract_patches_around_landmarks(group='source', as_single_array=True,
patch_shape=svm_p['patch_s']).flatten()
if svm_p['clf'].decision_function(_p_nd) > 0:
copy2(p_wr, clip.path_write_ln[1] + im.path.stem + '_0.pts')
def test_importing_v3_ljson_null_values(is_file, mock_open, mock_dict):
v3_ljson = {
"groups": {
"LJSON": {
"labels": [{
"label": "left_eye",
"mask": [0, 1, 2]
}, {
"label": "right_eye",
"mask": [3, 4, 5]
}],
"landmarks": {
"connectivity": [[0, 1], [1, 2], [2, 0], [3, 4], [4, 5],
[5, 3]],
"points": [[None, 200.5], [None, None], [316.8, 199.15],
[339.48, 205.0], [358.54, 217.82],
[375.0, 233.4]]
}
}
},
"version": 3
}
mock_dict.return_value = v3_ljson
is_file.return_value = True
lmark_dict = mio.import_landmark_file('fake_lmark_being_mocked.ljson')
assert isinstance(lmark_dict, dict)
lmark = lmark_dict['LJSON']
nan_points = np.isnan(lmark.points)
assert nan_points[0, 0] # y-coord None point is nan
assert not nan_points[0, 1] # x-coord point is not nan
assert np.all(nan_points[1, :]) # all of leye label is nan
def main():
path_to_neutral = Path(
'/Users/lls/Documents/face/data/headpose/Angle/neutral0/')
path_to_smile = Path(
'/Users/lls/Documents/face/data/headpose/Angle/down30')
path_to_source = Path('/Users/lls/Documents/face/data/trump/trump')
# PDM shape
neutral = PDMModel(path_to_neutral, 20)[0].model.mean()
smile = PDMModel(path_to_smile, 20)[0].model.mean()
source_shape, source_img = PDMModel(path_to_source, 20)
p_smile = project(smile, source_shape)
p_neutral = project(neutral, source_shape)
delta = (p_smile - p_neutral) * 1.5
ptsPath = '/Users/lls/Documents/face/data/trump/trump/trump_13.pts'
trumpShape = mio.import_landmark_file(ptsPath).lms
p_i = project(trumpShape, source_shape)
new_p_i = p_i + delta
reconstructed_img_i = source_shape.model.instance(new_p_i)
trans_reconstructed_img_i = AlignmentAffine(reconstructed_img_i,
trumpShape)
reconstructed_img_i_pc = trans_reconstructed_img_i.apply(
reconstructed_img_i)
plt.subplot(241)
reconstructed_img_i_pc.view()
plt.gca().set_title('reconstructed_img_i_pc')
plt.subplot(242)
trumpShape.view()
plt.gca().set_title('trumpShape')
plt.show()
'''
def test_importing_v2_ljson_null_values(is_file, mock_open, mock_dict):
v2_ljson = {
"labels": [{
"label": "left_eye",
"mask": [0, 1, 2]
}, {
"label": "right_eye",
"mask": [3, 4, 5]
}],
"landmarks": {
"connectivity": [[0, 1], [1, 2], [2, 0], [3, 4], [4, 5], [5, 3]],
"points": [[None, 200.5], [None, None], [316.8, 199.15],
[339.48, 205.0], [358.54, 217.82], [375.0, 233.4]]
},
"version":
2
}
mock_dict.return_value = v2_ljson
is_file.return_value = True
with warnings.catch_warnings(record=True) as w:
lmark = mio.import_landmark_file('fake_lmark_being_mocked.ljson',
group='LJSON')
nan_points = np.isnan(lmark.points)
assert nan_points[0, 0] # y-coord None point is nan
assert not nan_points[0, 1] # x-coord point is not nan
assert np.all(nan_points[1, :]) # all of leye label is nan
def test():
img = mio.import_image(
'/home/sean/workplace/221/py-R-FCN-test/data/DB/face/300-w_face/otherDB/aflw-full/testset/0_image00002_1.jpg'
)
if img.n_channels != 1:
img = img.as_greyscale()
img.landmarks['face'] = mio.import_landmark_file(
'/home/sean/workplace/221/py-R-FCN-test/data/DB/face/300-w_face/temp/indoor_001.pts'
)
# objects return copies rather than mutating self, so we can chain calls
img = (img.crop_to_landmarks(
group='face',
boundary=10).rescale_landmarks_to_diagonal_range(100, group='face'))
# now lets take an image feature...
img = fast_dsift(img)
# ...and extract the vector of pixels contained in the
# convex hull of the face...
vector = img.as_masked().constrain_mask_to_landmarks(
group='face').as_vector()
print(type(vector), vector.shape)
def serialize_sample(writer, subject_id):
subject_name = 'P{}'.format(subject_id)
for i, (video, audio, label) in enumerate(zip(*get_samples(subject_name))):
frame = Image.init_from_channels_at_back(video)
lms_path = landmarks_directory / subject_name / "{}.pts".format(i)
try:
lms = mio.import_landmark_file(lms_path)
except Exception as e:
print('Landmark file [{}] could not be imported'.format(i))
print('Exception message : {}'.format(e))
continue
frame.landmarks['PTS'] = lms
frame = crop_face(frame)
example = tf.train.Example(features=tf.train.Features(
feature={
'sample_id': _int_feauture(i),
'subject_id': _int_feauture(subject_id),
'label': _bytes_feauture(label.tobytes()),
'raw_audio': _bytes_feauture(audio.tobytes()),
'frame': _bytes_feauture(get_jpg_string(frame))
}))
writer.write(example.SerializeToString())
del video, audio, label
def test_json_landmarks_bunny_direct():
lms = mio.import_landmark_file(mio.data_path_to('bunny.ljson'))
labels = {'reye', 'mouth', 'nose', 'leye'}
assert(len(labels - set(lms.labels)) == 0)
assert_allclose(lms['leye'].points, bunny_leye, atol=1e-7)
assert_allclose(lms['reye'].points, bunny_reye, atol=1e-7)
assert_allclose(lms['nose'].points, bunny_nose, atol=1e-7)
assert_allclose(lms['mouth'].points, bunny_mouth, atol=1e-7)
def test_json_landmarks_bunny_direct():
lms = pio.import_landmark_file(pio.data_path_to('bunny.json'))
assert(lms.group_label == 'JSON')
labels = {'r_eye', 'mouth', 'nose', 'l_eye'}
assert(len(labels - set(lms.labels)) == 0)
assert_allclose(lms['l_eye'].lms.points, bunny_l_eye, atol=1e-7)
assert_allclose(lms['r_eye'].lms.points, bunny_r_eye, atol=1e-7)
assert_allclose(lms['nose'].lms.points, bunny_nose, atol=1e-7)
assert_allclose(lms['mouth'].lms.points, bunny_mouth, atol=1e-7)
def test_json_landmarks_bunny_direct():
lms = pio.import_landmark_file(pio.data_path_to('bunny.json'))
assert (lms.group_label == 'JSON')
labels = {'r_eye', 'mouth', 'nose', 'l_eye'}
assert (len(labels - set(lms.labels)) == 0)
assert_allclose(lms['l_eye'].lms.points, bunny_l_eye, atol=1e-7)
assert_allclose(lms['r_eye'].lms.points, bunny_r_eye, atol=1e-7)
assert_allclose(lms['nose'].lms.points, bunny_nose, atol=1e-7)
assert_allclose(lms['mouth'].lms.points, bunny_mouth, atol=1e-7)
def load_images_test(paths,
reference_shape,
group=None,
verbose=True,
PLOT=False):
"""Loads and rescales input knn_2D to the diagonal of the reference shape.
Args:
paths: a list of strings containing the data directories.
reference_shape (meanshape): a numpy array [num_landmarks, 2]
group: landmark group containing the grounth truth landmarks.
verbose: boolean, print debugging info.
Returns:
knn_2D: a list of numpy arrays containing knn_2D.
shapes: a list of the ground truth landmarks.
reference_shape (meanshape): a numpy array [num_landmarks, 2].
shape_gen: PCAModel, a shape generator.
"""
images = []
shapes = []
scales = []
# compute mean shape
reference_shape = PointCloud(reference_shape)
nameList = []
bbox = []
data = dict()
for path in paths:
if verbose:
print('Importing data from {}'.format(path))
for im in mio.import_images(path, verbose=verbose, as_generator=True):
# group = group or im.landmarks[group]._group_label
group = group or im.landmarks.keys()[0]
bb_root = im.path.parent.relative_to(im.path.parent.parent.parent)
if 'set' not in str(bb_root):
bb_root = im.path.parent.relative_to(im.path.parent.parent)
im.landmarks['bb'] = mio.import_landmark_file(
str(
Path('bbs') / bb_root /
(im.path.stem.replace(' ', '') + '.pts')))
nameList.append(str(im.path))
lms = im.landmarks['bb'].lms.points
bbox.append([lms[0, 1], lms[2, 1], lms[0, 0], lms[1, 0]])
# bbox = np.array(bbox)
# data['nameList'] = nameList
# data['bbox'] = bbox
# sio.savemat('ibug_data.mat', {'nameList':data['nameList'], 'bbox':data['bbox']})
# exit(0)
im = im.crop_to_landmarks_proportion(0.3, group='bb')
images.append(im)
return images
def import_image(img_path):
img = cv2.imread(str(img_path))
original_image = Image.init_from_channels_at_back(img[:, :, -1::-1])
try:
original_image_lms = mio.import_landmark_file('{}/{}.ljson'.format(
img_path.parent, img_path.stem)).lms.points.astype(np.float32)
original_image.landmarks['LJSON'] = PointCloud(original_image_lms)
except:
pass
return original_image
def test_register_landmark_importer(is_file):
from menpo.shape import PointCloud
lmark = PointCloud.init_2d_grid((1, 1))
def foo_importer(filepath, **kwargs):
return lmark
is_file.return_value = True
with patch.dict(mio.input.extensions.image_landmark_types, {}, clear=True):
mio.register_landmark_importer('.foo', foo_importer)
new_lmark = mio.import_landmark_file('fake.foo')
assert lmark is new_lmark
def test_register_landmark_importer(is_file):
from menpo.shape import PointCloud
lmark = PointCloud.init_2d_grid((1, 1))
def foo_importer(filepath, **kwargs):
return lmark
is_file.return_value = True
with patch.dict(mio.input.extensions.image_landmark_types, {}, clear=True):
mio.register_landmark_importer('.foo', foo_importer)
new_lmark = mio.import_landmark_file('fake.foo')
assert lmark is new_lmark
def load_image(path,
reference_shape,
is_training=False,
group='PTS',
mirror_image=False):
"""Load an annotated image.
In the directory of the provided image file, there
should exist a landmark file (.pts) with the same
basename as the image file.
Args:
path: a path containing an image file.
reference_shape: a numpy array [num_landmarks, 2]
is_training: whether in training mode or not.
group: landmark group containing the grounth truth landmarks.
mirror_image: flips horizontally the image's pixels and landmarks.
Returns:
pixels: a numpy array [width, height, 3].
estimate: an initial estimate a numpy array [68, 2].
gt_truth: the ground truth landmarks, a numpy array [68, 2].
"""
im = mio.import_image(path)
bb_root = im.path.parent.relative_to(im.path.parent.parent.parent)
if 'set' not in str(bb_root):
bb_root = im.path.parent.relative_to(im.path.parent.parent)
im.landmarks['bb'] = mio.import_landmark_file(
str(Path('bbs') / bb_root / (im.path.stem + '.pts')))
im = im.crop_to_landmarks_proportion(0.3, group='bb')
reference_shape = PointCloud(reference_shape)
bb = im.landmarks['bb'].lms.bounding_box()
im.landmarks['__initial'] = align_shape_with_bounding_box(
reference_shape, bb)
im = im.rescale_to_pointcloud(reference_shape, group='__initial')
if mirror_image:
im = utils.mirror_image(im)
lms = im.landmarks[group].lms
initial = im.landmarks['__initial'].lms
# if the image is greyscale then convert to rgb.
pixels = grey_to_rgb(im).pixels.transpose(1, 2, 0)
gt_truth = lms.points.astype(np.float32)
estimate = initial.points.astype(np.float32)
return pixels.astype(np.float32).copy(), gt_truth, estimate
def load_images(list_frames, frames_path, path_land, clip_name, max_images=None,
training_images=None, crop_reading=0.3, pix_thres=330, feat=None):
"""
Read images from the clips that are processed. The landmarks can be a different folder with the extension of pts and
are searched as such.
:param list_frames: List of images that will be read and loaded.
:param frames_path: Path to the folder of images.
:param path_land: Path of the respective landmarks.
:param clip_name: The name of the clip being processed.
:param max_images: (optional) Max images that will be loaded from this clip.
:param training_images: (optional) List of images to append the new ones.
:param crop_reading: (optional) Amount of cropping the image around the landmarks.
:param pix_thres: (optional) If the cropped image has a dimension bigger than this, it gets cropped to this diagonal dimension.
:param feat: (optional) Features to be applied to the images before inserting them to the list.
:return: List of menpo images.
"""
from random import shuffle
if not check_path_and_landmarks(frames_path, clip_name, path_land):
return []
if feat is None:
feat = no_op
if training_images is None:
training_images = []
shuffle(list_frames) # shuffle the list to ensure random ones are chosen
if max_images is None:
max_images = len(list_frames)
elif max_images < 0:
print('Warning: The images cannot be negative, loading the whole list instead.')
max_images = len(list_frames)
cnt = 0 # counter for images appended to the list
for frame_name in list_frames:
try:
im = mio.import_image(frames_path + frame_name, normalise=True)
except ValueError: # in case the extension is unknown (by menpo)
print('Ignoring the \'image\' {}.'.format(frame_name))
continue
res = glob.glob(path_land + clip_name + sep + im.path.stem + '*.pts')
if len(res) == 0: # if the image does not have any existing landmarks, ignore it
continue
elif len(res) > 1:
#_r = randint(0,len(res)-1); #just for debugging reasons in different variable
#ln = mio.import_landmark_file(res[_r]) # in case there are plenty of landmarks for the image, load random ones
print('The image {} has more than one landmarks, for one person, loading only the first ones.'.format(frame_name))
ln = mio.import_landmark_file(res[0])
im.landmarks['PTS'] = ln
im = crop_rescale_img(im, crop_reading=crop_reading, pix_thres=pix_thres)
training_images.append(feat(im))
cnt += 1
if cnt >= max_images:
break # the limit of images (appended to the list) is reached
return training_images
def test_importing_v1_ljson_null_values(is_file, mock_open, mock_dict):
v1_ljson = {
"groups": [
{
"connectivity": [[0, 1], [1, 2], [2, 3]],
"label":
"chin",
"landmarks": [
{
"point": [987.9, 1294.1]
},
{
"point": [96.78, 1246.8]
},
{
"point": [None, 0.1]
},
{
"point": [303.22, 167.2]
},
],
},
{
"connectivity": [[0, 1]],
"label": "leye",
"landmarks": [{
"point": [None, None]
}, {
"point": [None, None]
}],
},
],
"version":
1,
}
mock_dict.return_value = v1_ljson
is_file.return_value = True
with warnings.catch_warnings(record=True) as found_warnings:
lmark = mio.import_landmark_file("fake_lmark_being_mocked.ljson",
group="LJSON")
nan_points = np.isnan(lmark.points)
# Should raise deprecation warning
assert len(found_warnings) == 1, [w.message for w in found_warnings]
assert nan_points[2, 0] # y-coord None point is nan
assert not nan_points[2, 1] # x-coord point is not nan
assert np.all(nan_points[4:, :]) # all of leye label is nan
def load_image(path, reference_shape, is_training=False, group='PTS',
mirror_image=False):
"""Load an annotated image.
In the directory of the provided image file, there
should exist a landmark file (.pts) with the same
basename as the image file.
Args:
path: a path containing an image file.
reference_shape: a numpy array [num_landmarks, 2]
is_training: whether in training mode or not.
group: landmark group containing the grounth truth landmarks.
mirror_image: flips horizontally the image's pixels and landmarks.
Returns:
pixels: a numpy array [width, height, 3].
estimate: an initial estimate a numpy array [68, 2].
gt_truth: the ground truth landmarks, a numpy array [68, 2].
"""
im = mio.import_image(path)
bb_root = im.path.parent.relative_to(im.path.parent.parent.parent)
if 'set' not in str(bb_root):
bb_root = im.path.parent.relative_to(im.path.parent.parent)
im.landmarks['bb'] = mio.import_landmark_file(str(Path('bbs') / bb_root / (
im.path.stem + '.pts')))
im = im.crop_to_landmarks_proportion(0.3, group='bb')
reference_shape = PointCloud(reference_shape)
bb = im.landmarks['bb'].lms.bounding_box()
im.landmarks['__initial'] = align_shape_with_bounding_box(reference_shape,
bb)
im = im.rescale_to_pointcloud(reference_shape, group='__initial')
if mirror_image:
im = utils.mirror_image(im)
lms = im.landmarks[group].lms
initial = im.landmarks['__initial'].lms
# if the image is greyscale then convert to rgb.
pixels = grey_to_rgb(im).pixels.transpose(1, 2, 0)
gt_truth = lms.points.astype(np.float32)
estimate = initial.points.astype(np.float32)
return pixels.astype(np.float32).copy(), gt_truth, estimate
def build_reference_shape(paths, num_patches=73, diagonal=200):
"""Builds the reference shape.
Args:
paths: train image paths.
num_patches: number of landmarks
diagonal: the diagonal of the reference shape in pixels.
Returns:
the reference shape.
"""
landmarks = []
for path in paths:
group = mio.import_landmark_file(path.parent / (path.stem + '.pts'))
if group.n_points == num_patches:
landmarks += [group]
return compute_reference_shape(landmarks, diagonal=diagonal).points.astype(np.float32)
def _aux(im, pts_paths, pts_names, pts_formats, save_path, save_original, off1, off2,
figure_size, overwrite, render_options, only_ln=False):
if only_ln: # case of visualising only landmarks (black background)
path_tmp = im.path
im = Image.init_blank([im.shape[0], im.shape[1]], im.n_channels)
im.path = path_tmp
# attach landmarks
for k, pts_path in enumerate(pts_paths):
if os.path.isfile(pts_path + im.path.stem + pts_formats[k]):
pts = mio.import_landmark_file(pts_path + im.path.stem + pts_formats[k])
im.landmarks[pts_names[k]] = pts
# copy original if asked
if save_original:
im_orig = im.copy()
# crop
if pts_names[0] in im.landmarks.group_labels:
centre = im.landmarks[pts_names[0]].lms.centre()
min_indices = np.array([round(centre[0])-off1, round(centre[1])-off2])
max_indices = np.array([round(centre[0])+off1, round(centre[1])+off2])
# im.crop_inplace(min_indices, max_indices)
im = im.crop(min_indices, max_indices, constrain_to_boundary=True)
else:
path_tmp = im.path
im = Image.init_blank([off1*2 + 1, off2*2 + 1], im.n_channels)
im.path = path_tmp
# render
rand = randint(1, 10000)
fig = plt.figure(rand)
if save_original:
gs = gridspec.GridSpec(1, 2, width_ratios=[im_orig.height, im.height])
plt.subplot(gs[0])
renderer = _render(im_orig, pts_names, fig, render_options['colours'][0],
render_options['sizes'][0], render_options['edgesizes'][0], figure_size)
plt.subplot(gs[1])
renderer = _render(im, pts_names, fig, render_options['colours'][1],
render_options['sizes'][1], render_options['edgesizes'][1], figure_size)
else:
renderer = _render(im, pts_names, fig, render_options['colours'][1],
render_options['sizes'][1], render_options['edgesizes'][1], figure_size)
renderer.save_figure(save_path + im.path.stem + '.png', format='png', pad_inches=0.0, overwrite=overwrite)
plt.close(rand)
def _aux(im, pts_paths, pts_names, pts_formats, save_path, save_original, off1, off2, figure_size, overwrite, render_options, only_ln=False):
if only_ln: # case of visualising only landmarks (black background)
path_tmp = im.path
im = Image.init_blank([im.shape[0], im.shape[1]], im.n_channels)
im.path = path_tmp
# attach landmarks
for k, pts_path in enumerate(pts_paths):
if os.path.isfile(pts_path + im.path.stem + pts_formats[k]):
pts = mio.import_landmark_file(pts_path + im.path.stem + pts_formats[k])
im.landmarks[pts_names[k]] = pts
# copy original if asked
if save_original:
im_orig = im.copy()
# crop
if pts_names[0] in im.landmarks.group_labels:
centre = im.landmarks[pts_names[0]].lms.centre()
min_indices = np.array([round(centre[0])-off1, round(centre[1])-off2])
max_indices = np.array([round(centre[0])+off1, round(centre[1])+off2])
# im.crop_inplace(min_indices, max_indices)
im = im.crop(min_indices, max_indices, constrain_to_boundary=True)
else:
path_tmp = im.path
im = Image.init_blank([off1*2 + 1, off2*2 + 1], im.n_channels)
im.path = path_tmp
# render
rand = randint(1, 10000)
fig = plt.figure(rand)
if save_original:
gs = gridspec.GridSpec(1, 2, width_ratios=[im_orig.height, im.height])
plt.subplot(gs[0])
renderer = _render(im_orig, pts_names, fig, render_options['colours'][0],
render_options['sizes'][0], render_options['edgesizes'][0], figure_size)
plt.subplot(gs[1])
renderer = _render(im, pts_names, fig, render_options['colours'][1],
render_options['sizes'][1], render_options['edgesizes'][1], figure_size)
else:
renderer = _render(im, pts_names, fig, render_options['colours'][1],
render_options['sizes'][1], render_options['edgesizes'][1], figure_size)
renderer.save_figure(save_path + im.path.stem + '.png', format='png', pad_inches=0.0, overwrite=overwrite)
plt.close(rand)
def wrapper(index, shape):
index = index.decode("utf-8")
prefix = index.split('_')[0]
landmark_indices = list(map(int, index.split('_')[1:]))
if len(landmark_indices) > 1:
min_index, max_index = landmark_indices
landmark_indices = range(min_index, max_index + 1)
kpts = np.zeros(shape[:2], dtype=int)
im = Image(kpts)
mask = np.ones(list(shape[:2]) + [1]).astype(np.float32)
for lms_index in landmark_indices:
filename = (
prefix + '_' + str(lms_index) + '.' + self.lms_extension)
path = self.lms_root / filename
if not path.exists():
continue
lms = mio.import_landmark_file(path.as_posix()).lms
if lms.points.shape[0] != 68:
min_indices, max_indices = lms.bounds()
mask[min_indices[0]:max_indices[0], min_indices[1]:
max_indices[1]] = 0
continue
for i in range(68):
lms_mask = im.as_masked().copy()
patches = np.ones((1, 1, 1, 4, 4), dtype=np.bool)
pc = lms.points[i][None, :]
lms_mask.mask.pixels[...] = False
lms_mask = lms_mask.mask.set_patches(
patches, menpo.shape.PointCloud(pc))
kpts[lms_mask.mask] = i + 1
return kpts.astype(np.int32), mask.astype(np.int32)
def test_importing_v2_ljson_null_values(is_file, mock_open, mock_dict):
v2_ljson = { "labels": [
{ "label": "left_eye", "mask": [0, 1, 2] },
{ "label": "right_eye", "mask": [3, 4, 5] }
],
"landmarks": {
"connectivity": [ [0, 1], [1, 2], [2, 0], [3, 4],
[4, 5], [5, 3] ],
"points": [ [None, 200.5], [None, None],
[316.8, 199.15], [339.48, 205.0],
[358.54, 217.82], [375.0, 233.4]]
},
"version": 2 }
mock_dict.return_value = v2_ljson
is_file.return_value = True
lmark = mio.import_landmark_file('fake_lmark_being_mocked.ljson')
nan_points = np.isnan(lmark.lms.points)
assert nan_points[0, 0] # y-coord None point is nan
assert not nan_points[0, 1] # x-coord point is not nan
assert np.all(nan_points[1, :]) # all of leye label is nan
def get_keys(self, path='images'):
path = self.root / path
lms_files = path.glob('*' + self.lms_extension)
keys = [] # ['face_55135', 'face_49348']
# Get only files with 68 landmarks
for p in lms_files:
try:
lms = mio.import_landmark_file(p)
if lms.n_landmarks == 68 and not np.isnan(lms.lms.points).any(
):
keys.append(lms.path.stem)
except:
pass
self._keys = keys
print('Found {} files.'.format(len(keys)))
if len(keys) == 0:
raise RuntimeError('No images found in {}'.format(path))
return tf.constant(keys, tf.string)
def test_importing_v1_ljson_null_values(is_file, mock_open, mock_dict):
v1_ljson = { "groups": [
{ "connectivity": [ [ 0, 1 ], [ 1, 2 ], [ 2, 3 ] ],
"label": "chin", "landmarks": [
{ "point": [ 987.9, 1294.1 ] }, { "point": [ 96.78, 1246.8 ] },
{ "point": [ None, 0.1 ] }, { "point": [303.22, 167.2 ] } ] },
{ "connectivity": [ [ 0, 1 ] ],
"label": "leye", "landmarks": [
{ "point": [ None, None ] },
{ "point": [ None, None ] }] }
], "version": 1 }
mock_dict.return_value = v1_ljson
is_file.return_value = True
with warnings.catch_warnings(record=True) as w:
lmark = mio.import_landmark_file('fake_lmark_being_mocked.ljson')
nan_points = np.isnan(lmark.lms.points)
# Should raise deprecation warning
assert len(w) == 1
assert nan_points[2, 0] # y-coord None point is nan
assert not nan_points[2, 1] # x-coord point is not nan
assert np.all(nan_points[4:, :]) # all of leye label is nan
def main():
trumpFolder = './data/trump/trump'
leftFolder = './data/headpose/Angle/left15'
rightFolder = './data/headpose/Angle/right15'
neutralFolder = './data/headpose/Angle/neutral0'
downFolder = './data/headpose/Angle/down30'
'''SHAPE MODEL'''
trumpShapeModel = PointDistributionModel(trumpFolder)
leftShapeModel = PointDistributionModel(leftFolder)
rightShapeModel = PointDistributionModel(rightFolder)
neutralShapeModel = PointDistributionModel(neutralFolder)
downShapeModel = PointDistributionModel(downFolder)
'''MEAN SHAPE'''
trumpMeanShape = trumpShapeModel.model.mean()
leftMeanShape = leftShapeModel.model.mean()
rightMeanShape = rightShapeModel.model.mean()
neutralMeanShape = neutralShapeModel.model.mean()
downMeanShape = downShapeModel.model.mean()
'''WEIGHTS'''
leftWeights = Projection(trumpShapeModel, leftMeanShape)
rightWeights = Projection(trumpShapeModel, rightMeanShape)
neutralWeights = Projection(trumpShapeModel, neutralMeanShape)
downWeights = Projection(trumpShapeModel, downMeanShape)
'''DELTA'''
leftDeltaWeights = deltaWeights(leftWeights, neutralWeights)
rightDeltaWeights = deltaWeights(rightWeights, neutralWeights)
downDeltaWeights = deltaWeights(downWeights, neutralWeights)
'''RECONSTRUCTION'''
'''PRODUCE TRUMP IMAGES'''
for root, dirs, files in os.walk(trumpFolder):
for file in files:
if os.path.join(root, file).endswith('.pts'):
'''READ ONE TRUMP IMAGE/PTS'''
ptsPath = os.path.join(root, file)
trumpShape = mio.import_landmark_file(ptsPath)
headTrumpShape = Reconstruction(trumpShapeModel,
leftDeltaWeights, trumpShape)
def _validate_without_gt(self, valid_imgs, fitter, vis=True):
Pxy_template = self.read_pts(self.template_filename) # x ~ y format
menpo_initial_pts = mio.import_landmark_file(
self.template_filename).lms
Pyx_template = self._swap_columns(Pxy_template)
for im in valid_imgs:
path = im.path._str
pathdir, name, ext = fp.fileparts(path)
Pyx_initial_pts = Pyx_template * np.array(
[im.pixels.shape[1], im.pixels.shape[2]], np.float32)
Pxy_initial_pts = self._swap_columns(Pyx_initial_pts)
menpo_initial_pts.points = Pyx_initial_pts
fr = fitter.fit_from_shape(im, menpo_initial_pts, gt_shape=None)
menpo_output_pts = fr.final_shape
Pxy_output_pts = self._swap_columns(menpo_output_pts.points)
self.save_pts(os.path.join(pathdir, name + '.pts'), Pxy_output_pts)
if vis is True:
Pxy_initial_pts = Pxy_initial_pts.astype(np.int32)
Pxy_output_pts = Pxy_output_pts.astype(np.int32)
im_vis = cv2.imread(path)
for k in range(Pxy_output_pts.shape[0]):
cv2.circle(im_vis,
(Pxy_initial_pts[k, 0], Pxy_initial_pts[k, 1]),
10, (0, 255, 0), 10) # green
cv2.circle(im_vis,
(Pxy_output_pts[k, 0], Pxy_output_pts[k, 1]),
10, (0, 0, 255), 10) # red
im_vis = cv2.resize(im_vis, dsize=(0, 0), fx=0.3, fy=0.3)
if not os.path.exists('./vis/'):
os.mkdir('./vis/')
cv2.imwrite(os.path.join('./vis/', name + '.jpg'), im_vis)
from numpy.testing import assert_allclose
import os
from pathlib import PosixPath, WindowsPath, Path
from mock import patch, PropertyMock, MagicMock
from nose.tools import raises
import menpo.io as mio
from menpo.io.utils import _norm_path
from menpo.image import Image
from menpo.io.output.pickle import pickle_paths_as_pure
builtins_str = '__builtin__' if sys.version_info[0] == 2 else 'builtins'
test_lg = mio.import_landmark_file(mio.data_path_to('lenna.ljson'))
nan_lg = test_lg.copy()
nan_lg.points[0, :] = np.nan
test_img = Image(np.random.random([100, 100]))
colour_test_img = Image(np.random.random([3, 100, 100]))
fake_path = '/tmp/test.fake'
@patch('menpo.io.output.base.landmark_types')
@patch('menpo.io.output.base.Path.exists')
@patch('menpo.io.output.base.Path.open')
def test_export_filepath_overwrite_exists(mock_open, exists, landmark_types):
exists.return_value = True
landmark_types.__contains__.return_value = True
mio.export_landmark_file(test_lg, fake_path, overwrite=True)
mock_open.assert_called_with('wb')
def test_import_landmark_file():
lm_path = os.path.join(mio.data_dir_path(), 'einstein.pts')
mio.import_landmark_file(lm_path)
def load_images(paths, group=None, verbose=True):
"""Loads and rescales input images to the diagonal of the reference shape.
Args:
paths: a list of strings containing the data directories.
reference_shape: a numpy array [num_landmarks, 2]
group: landmark group containing the grounth truth landmarks.
verbose: boolean, print debugging info.
Returns:
images: a list of numpy arrays containing images.
shapes: a list of the ground truth landmarks.
reference_shape: a numpy array [num_landmarks, 2].
shape_gen: PCAModel, a shape generator.
"""
images = []
shapes = []
bbs = []
reference_shape = PointCloud(build_reference_shape(paths))
for path in paths:
if verbose:
print('Importing data from {}'.format(path))
for im in mio.import_images(path, verbose=verbose, as_generator=True):
group = group or im.landmarks[group]._group_label
bb_root = im.path.parent.relative_to(im.path.parent.parent.parent)
if 'set' not in str(bb_root):
bb_root = im.path.parent.relative_to(im.path.parent.parent)
im.landmarks['bb'] = mio.import_landmark_file(str(Path(
'bbs') / bb_root / (im.path.stem + '.pts')))
im = im.crop_to_landmarks_proportion(0.3, group='bb')
im = im.rescale_to_pointcloud(reference_shape, group=group)
im = grey_to_rgb(im)
images.append(im.pixels.transpose(1, 2, 0))
shapes.append(im.landmarks[group].lms)
bbs.append(im.landmarks['bb'].lms)
train_dir = Path(FLAGS.train_dir)
mio.export_pickle(reference_shape.points, train_dir / 'reference_shape.pkl', overwrite=True)
print('created reference_shape.pkl using the {} group'.format(group))
pca_model = detect.create_generator(shapes, bbs)
# Pad images to max length
max_shape = np.max([im.shape for im in images], axis=0)
max_shape = [len(images)] + list(max_shape)
padded_images = np.random.rand(*max_shape).astype(np.float32)
print(padded_images.shape)
for i, im in enumerate(images):
height, width = im.shape[:2]
dy = max(int((max_shape[1] - height - 1) / 2), 0)
dx = max(int((max_shape[2] - width - 1) / 2), 0)
lms = shapes[i]
pts = lms.points
pts[:, 0] += dy
pts[:, 1] += dx
lms = lms.from_vector(pts)
padded_images[i, dy:(height+dy), dx:(width+dx)] = im
return padded_images, shapes, reference_shape.points, pca_model
def influence():
image_paths = sorted(list(Path('.').glob(FLAGS.dataset)))
with tf.Graph().as_default(), tf.device('/cpu:0'):
with open(MDM_MODEL_PATH, 'rb') as f:
graph_def = tf.GraphDef.FromString(f.read())
tf.import_graph_def(graph_def)
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
errors = []
mean_errors = []
step = 0
start_time = time.time()
for path in image_paths:
mp_image = mio.import_image(path)
assert isinstance(mp_image, menpo.image.Image)
if mp_image.n_channels == 3:
mp_image.pixels = np.mean(mp_image.pixels,
0,
keepdims=True)
mp_image.landmarks['bb'] = mio.import_landmark_file(
str(
Path(path.parent.parent / 'BoundingBoxes' /
(path.stem + '.pts'))))
ly, lx = mp_image.landmarks['bb'].points[0]
hy, hx = mp_image.landmarks['bb'].points[2]
cx = (lx + hx) / 2
cy = (ly + hy) / 2
bb_size = int(math.ceil(max(hx - lx, hy - ly) * 4. / 6.))
square_bb = np.array([[cy - bb_size, cx - bb_size],
[cy + bb_size, cx + bb_size]])
mp_image.landmarks['square_bb'] = PointCloud(square_bb)
mp_image = mp_image.crop_to_landmarks_proportion(
0.0, group='square_bb')
mp_image = mp_image.resize((112, 112))
np_image = np.expand_dims(mp_image.pixels.transpose((1, 2, 0)),
0)
np_shape = mp_image.landmarks['PTS'].points
prediction, = sess.run('import/add:0',
feed_dict={'import/input:0': np_image})
assert isinstance(prediction, np.ndarray)
prediction = prediction.reshape((68, 2))
prediction = prediction[:, [1, 0]]
error = normalized_batch_nme(prediction,
mp_image.landmarks['PTS'].points)
mean_error = normalized_nme(error)
error_level = min(9, int(mean_error * 100))
concat_image = utils.draw_landmarks_discrete(
np_image[0], np_shape, prediction)
# plt.imsave('err{}/step{}.png'.format(error_level, step), concat_image)
errors.append(error)
mean_errors.append(mean_error)
step += 1
if step % 20 == 0:
duration = time.time() - start_time
sec_per_batch = duration / 20.0
examples_per_sec = 1. / sec_per_batch
log_str = '{}: [{:d} batches done] ({:.1f} examples/sec; {:.3f} sec/batch)'
print(
log_str.format(datetime.now(), step, examples_per_sec,
sec_per_batch))
start_time = time.time()
errors = np.array(errors)
print(errors.shape)
mean_errors = np.vstack(mean_errors).ravel()
errors_mean = np.mean(errors, 0)
mean_errors_mean = mean_errors.mean()
with open('errors.txt', 'w') as ofs:
for row, avg in zip(errors, mean_errors):
for col in row:
ofs.write('%.4f, ' % col)
ofs.write('%.4f' % avg)
ofs.write('\n')
for col in errors_mean:
ofs.write('%.4f, ' % col)
ofs.write('%.4f' % mean_errors_mean)
ofs.write('\n')
auc_at_08 = (mean_errors < .08).mean()
auc_at_05 = (mean_errors < .05).mean()
print('Errors', mean_errors.shape)
print(
'%s: mean_rmse = %.4f, auc @ 0.05 = %.4f, auc @ 0.08 = %.4f' %
(datetime.now(), mean_errors.mean(), auc_at_05, auc_at_08))
def test_import_landmark_file():
lm_path = mio.data_dir_path() / 'einstein.pts'
mio.import_landmark_file(lm_path)
def load_images_aflw(paths,
group=None,
verbose=True,
PLOT=True,
AFLW=False,
PLOT_shape=False):
"""Loads and rescales input knn_2D to the diagonal of the reference shape.
Args:
paths: a list of strings containing the data directories.
reference_shape (meanshape): a numpy array [num_landmarks, 2]
group: landmark group containing the grounth truth landmarks.
verbose: boolean, print debugging info.
Returns:
knn_2D: a list of numpy arrays containing knn_2D.
shapes: a list of the ground truth landmarks.
reference_shape (meanshape): a numpy array [num_landmarks, 2].
shape_gen: PCAModel, a shape generator.
"""
images = []
shapes = []
bbs = []
shape_space = []
plot_shape_x = []
plot_shape_y = []
# compute mean shape
if AFLW:
# reference_shape = PointCloud(mio.import_pickle(Path('/home/hliu/gmh/RL_FA/mdm_aflw/ckpt/train_aflw') / 'reference_shape.pkl'))
reference_shape = mio.import_pickle(
Path('/home/hliu/gmh/RL_FA/mdm_aflw/ckpt/train_aflw') /
'reference_shape.pkl')
else:
reference_shape = PointCloud(build_reference_shape(paths))
for path in paths:
if verbose:
print('Importing data from {}'.format(path))
for im in mio.import_images(path, verbose=verbose, as_generator=True):
# group = group or im.landmarks[group]._group_label
group = group or im.landmarks.keys()[0]
bb_root = im.path.parent.relative_to(im.path.parent.parent.parent)
if 'set' not in str(bb_root):
bb_root = im.path.parent.relative_to(im.path.parent.parent)
if AFLW:
im.landmarks['bb'] = im.landmarks['PTS'].lms.bounding_box()
else:
im.landmarks['bb'] = mio.import_landmark_file(
str(Path('bbs') / bb_root / (im.path.stem + '.pts')))
im = im.crop_to_landmarks_proportion(0.3, group='bb')
im = im.rescale_to_pointcloud(reference_shape, group=group)
im = grey_to_rgb(im)
# knn_2D.append(im.pixels.transpose(1, 2, 0))
shapes.append(im.landmarks[group].lms)
shape_space.append(im.landmarks[group].lms.points)
bbs.append(im.landmarks['bb'].lms)
if PLOT_shape:
x_tmp = np.sum((im.landmarks[group].lms.points[:, 0] -
reference_shape.points[:, 0]))
y_tmp = np.sum((im.landmarks[group].lms.points[:, 1] -
reference_shape.points[:, 1]))
if x_tmp < 0 and y_tmp < 0:
plot_shape_x.append(x_tmp)
plot_shape_y.append(y_tmp)
shape_space = np.array(shape_space)
print('shape_space:', shape_space.shape)
train_dir = Path(FLAGS.train_dir)
if PLOT_shape:
k_nn_plot_x = []
k_nn_plot_y = []
centers = utils.k_means(shape_space, 500, num_patches=19)
centers = np.reshape(centers, [-1, 19, 2])
for i in range(centers.shape[0]):
x_tmp = np.sum((centers[i, :, 0] - reference_shape.points[:, 0]))
y_tmp = np.sum((centers[i, :, 1] - reference_shape.points[:, 1]))
if x_tmp < 0 and y_tmp < 0:
k_nn_plot_x.append(x_tmp)
k_nn_plot_y.append(y_tmp)
# plt.scatter(plot_shape_x, plot_shape_y, s=20)
# plt.scatter(k_nn_plot_x, k_nn_plot_y, s=40)
# plt.xticks(())
# plt.yticks(())
# plt.show()
# pdb.set_trace()
np.save(train_dir / 'shape_space_all.npy', shape_space)
# centers = utils.k_means(shape_space, 100)
# centers = np.reshape(centers, [-1, 68, 2])
# np.save(train_dir/'shape_space_origin.npy', centers)
# print('created shape_space.npy using the {} group'.format(group))
# exit(0)
mio.export_pickle(reference_shape.points,
train_dir / 'reference_shape.pkl',
overwrite=True)
print('created reference_shape.pkl using the {} group'.format(group))
pca_model = detect.create_generator(shapes, bbs)
# Pad knn_2D to max length
max_shape = [272, 261, 3]
padded_images = np.random.rand(*max_shape).astype(np.float32)
print(padded_images.shape)
if PLOT:
# plot without padding
centers = utils.k_means(shape_space, 500, num_patches=19)
centers = np.reshape(centers, [-1, 19, 2])
plot_img = cv2.imread('a.png').transpose(2, 0, 1)
centers_tmp = np.zeros(centers.shape)
# menpo_img = mio.import_image('a.png')
menpo_img = menpo.image.Image(plot_img)
for i in range(centers.shape[0]):
menpo_img.view()
min_y = np.min(centers[i, :, 0])
min_x = np.min(centers[i, :, 1])
centers_tmp[i, :, 0] = centers[i, :, 0] - min_y + 20
centers_tmp[i, :, 1] = centers[i, :, 1] - min_x + 20
print(centers_tmp[i, :, :])
menpo_img.landmarks['center'] = PointCloud(centers_tmp[i, :, :])
menpo_img.view_landmarks(group='center',
marker_face_colour='b',
marker_size='16')
# menpo_img.landmarks['center'].view(render_legend=True)
plt.savefig('plot_shape_space_aflw/' + str(i) + '.png')
plt.close()
exit(0)
# !!!shape_space without delta, which means shape_space has already been padded!
# delta = np.zeros(shape_space.shape)
for i, im in enumerate(images):
height, width = im.shape[:2]
dy = max(int((max_shape[0] - height - 1) / 2), 0)
dx = max(int((max_shape[1] - width - 1) / 2), 0)
lms = shapes[i]
pts = lms.points
pts[:, 0] += dy
pts[:, 1] += dx
shape_space[i, :, 0] += dy
shape_space[i, :, 1] += dx
# delta[i][:, 0] = dy
# delta[i][:, 1] = dx
lms = lms.from_vector(pts)
padded_images[i, dy:(height + dy), dx:(width + dx)] = im
# shape_space = np.concatenate((shape_space, delta), 2)
centers = utils.k_means(shape_space, 1000, num_patches=19)
centers = np.reshape(centers, [-1, 19, 2])
# pdb.set_trace()
np.save(train_dir / 'shape_space.npy', centers)
print('created shape_space.npy using the {} group'.format(group))
exit(0)
return padded_images, shapes, reference_shape.points, pca_model, centers
import warnings
from pathlib import Path, PosixPath, WindowsPath
from unittest.mock import MagicMock, PropertyMock, patch
import numpy as np
from numpy.testing import assert_allclose
from pytest import raises
import menpo.io as mio
from menpo.image import Image
from menpo.io.output.pickle import pickle_paths_as_pure
from menpo.io.utils import _norm_path
builtins_str = "__builtin__" if sys.version_info[0] == 2 else "builtins"
test_lg = mio.import_landmark_file(mio.data_path_to("lenna.ljson"),
group="LJSON")
nan_lg = test_lg.copy()
nan_lg.points[0, :] = np.nan
test_img = Image(np.random.random([100, 100]))
colour_test_img = Image(np.random.random([3, 100, 100]))
fake_path = "/tmp/test.fake"
@patch("menpo.io.output.base.landmark_types")
@patch("menpo.io.output.base.Path.exists")
@patch("menpo.io.output.base.Path.open")
def test_export_filepath_overwrite_exists(mock_open, exists, landmark_types):
exists.return_value = True
landmark_types.__contains__.return_value = True
mio.export_landmark_file(test_lg, fake_path, overwrite=True)
mock_open.assert_called_with("wb")
def test_import_landmark_file():
lm_path = mio.data_dir_path() / 'einstein.pts'
mio.import_landmark_file(lm_path)
def load_images_test_300VW(paths,
reference_shape,
group=None,
verbose=True,
PLOT=False):
"""Loads and rescales input knn_2D to the diagonal of the reference shape.
Args:
paths: a list of strings containing the data directories.
reference_shape (meanshape): a numpy array [num_landmarks, 2]
group: landmark group containing the grounth truth landmarks.
verbose: boolean, print debugging info.
Returns:
knn_2D: a list of numpy arrays containing knn_2D.
shapes: a list of the ground truth landmarks.
reference_shape (meanshape): a numpy array [num_landmarks, 2].
shape_gen: PCAModel, a shape generator.
"""
images = []
shapes = []
scales = []
# compute mean shape
reference_shape = PointCloud(reference_shape)
for path in paths:
if verbose:
print('Importing data from {}'.format(path))
for im in mio.import_images(path, verbose=verbose, as_generator=True):
# group = group or im.landmarks[group]._group_label
# pdb.set_trace()
# bb_root = im.path.parent.relative_to(im.path.parent.parent.parent)
bb_root = im.path.parent
if 'set' not in str(bb_root):
bb_root = im.path.parent.relative_to(im.path.parent.parent)
im.landmarks['bb'] = mio.import_landmark_file(
bb_root / str(Path('bbs') / (im.path.stem + '.pts')))
im.landmarks['PTS'] = mio.import_landmark_file(
bb_root / str(Path('annot') / (im.path.stem + '.pts')))
im = im.crop_to_landmarks_proportion(0.3, group='bb')
# im = im.rescale_to_pointcloud(reference_shape, group=group)
# _, height, width = im.pixels.shape
# im = im.resize([386, 458])
# im = grey_to_rgb(im)
# knn_2D.append(im.pixels.transpose(1, 2, 0))
# shapes.append(im.landmarks[group].lms.points.astype('float32'))
# scales.append([386/height, 485/width])
# lms = im.landmarks[group].lms
# im = im.pixels.transpose(1, 2, 0)
# height, width = im.shape[:2]
# # print('shape:', height, width)
# padded_image = np.random.rand(386, 458, 3).astype(np.float32)
# dy = max(int((386 - height - 1) / 2), 0)
# dx = max(int((458 - width - 1) / 2), 0)
# pts = lms.points
# pts[:, 0] += dy
# pts[:, 1] += dx
# # delta[i][:, 0] = dy
# # delta[i][:, 1] = dx
# lms = lms.from_vector(pts)
# padded_image[dy:(height+dy), dx:(width+dx), :] = im
images.append(im)
# shapes.append(lms.points.astype('float32'))
return images
def prepare_images(paths, num_patches=73, verbose=True):
"""Save Train Images to TFRecord
Args:
paths: a list of strings containing the data directories.
num_patches: number of landmarks
verbose: boolean, print debugging info.
Returns:
None
"""
if len(paths) == 0:
return
# .../<Dataset>/Images/*.png -> .../<Dataset>
path_base = Path(paths[0]).parent.parent
image_paths = []
# First: get all image paths
for path in paths:
for file in Path('.').glob(path):
try:
mio.import_landmark_file(
str(Path(file.parent.parent / 'BoundingBoxes' / (file.stem + '.pts')))
)
except ValueError:
continue
image_paths.append(file)
print('Got all image paths...')
# Second: split to train, test and validate. 7:2:1
if Path(path_base / 'train_img.txt').exists():
with Path(path_base / 'train_img.txt').open('rb') as train_ifs, \
Path(path_base / 'test_img.txt').open('rb') as test_ifs, \
Path(path_base / 'val_img.txt').open('rb') as val_ifs:
train_paths = [Path(line[:-1].decode('utf-8')) for line in train_ifs.readlines()]
test_paths = [Path(line[:-1].decode('utf-8')) for line in test_ifs.readlines()]
val_paths = [Path(line[:-1].decode('utf-8')) for line in val_ifs.readlines()]
else:
random.shuffle(image_paths)
num_train = int(len(image_paths) * 0.7)
num_test = int(len(image_paths) * 0.2)
train_paths = sorted(image_paths[:num_train])
test_paths = sorted(image_paths[num_train:num_train+num_test])
val_paths = sorted(image_paths[num_train+num_test:])
with Path(path_base / 'train_img.txt').open('wb') as train_ofs, \
Path(path_base / 'test_img.txt').open('wb') as test_ofs, \
Path(path_base / 'val_img.txt').open('wb') as val_ofs:
train_ofs.writelines([str(line).encode('utf-8') + b'\n' for line in train_paths])
test_ofs.writelines([str(line).encode('utf-8') + b'\n' for line in test_paths])
val_ofs.writelines([str(line).encode('utf-8') + b'\n' for line in val_paths])
print('Found Train/Test/Validate {}/{}/{}'.format(len(train_paths), len(test_paths), len(val_paths)))
# Third: export reference shape on train
if Path(path_base / 'reference_shape.pkl').exists():
reference_shape = PointCloud(mio.import_pickle(path_base / 'reference_shape.pkl'))
else:
reference_shape = PointCloud(build_reference_shape(train_paths, num_patches))
mio.export_pickle(reference_shape.points, path_base / 'reference_shape.pkl', overwrite=True)
print('Created reference_shape.pkl')
# Fourth: image shape & pca
image_shape = [0, 0, 3] # [H, W, C]
if Path(path_base / 'pca.bin').exists() and Path(path_base / 'meta.txt').exists():
with Path(path_base / 'meta.txt').open('r') as ifs:
image_shape = [int(x) for x in ifs.read().split(' ')]
else:
with tf.io.TFRecordWriter(str(path_base / 'pca.bin')) as ofs:
counter = 0
for path in train_paths:
counter += 1
if verbose:
status = 10.0 * counter / len(train_paths)
status_str = '\rPreparing {:2.2f}%['.format(status * 10)
for i in range(int(status)):
status_str += '='
for i in range(int(status), 10):
status_str += ' '
status_str += '] {} '.format(path)
print(status_str, end='')
mp_image = mio.import_image(path)
mp_image.landmarks['bb'] = mio.import_landmark_file(
str(Path(mp_image.path.parent.parent / 'BoundingBoxes' / (mp_image.path.stem + '.pts')))
)
mp_image = mp_image.crop_to_landmarks_proportion(0.3, group='bb')
mp_image = mp_image.rescale_to_pointcloud(reference_shape, group='PTS')
mp_image = grey_to_rgb(mp_image)
assert(mp_image.pixels.shape[0] == image_shape[2])
image_shape[0] = max(mp_image.pixels.shape[1], image_shape[0])
image_shape[1] = max(mp_image.pixels.shape[2], image_shape[1])
features = tf.train.Features(
feature={
'pca/shape': tf.train.Feature(
float_list=tf.train.FloatList(value=mp_image.landmarks['PTS'].points.flatten())
),
'pca/bb': tf.train.Feature(
float_list=tf.train.FloatList(value=mp_image.landmarks['bb'].points.flatten())
),
}
)
ofs.write(tf.train.Example(features=features).SerializeToString())
if verbose:
print('')
with Path(path_base / 'meta.txt').open('w') as ofs:
for s in image_shape[:-1]:
ofs.write('{} '.format(s))
ofs.write('{}'.format(image_shape[-1]))
print('Image shape', image_shape)
# Fifth: train data
if Path(path_base / 'train.bin').exists():
pass
else:
random.shuffle(train_paths)
with tf.io.TFRecordWriter(str(path_base / 'train.bin')) as ofs:
print('Preparing train data...')
counter = 0
for path in train_paths:
counter += 1
if verbose:
status = 10.0 * counter / len(train_paths)
status_str = '\rPreparing {:2.2f}%['.format(status * 10)
for i in range(int(status)):
status_str += '='
for i in range(int(status), 10):
status_str += ' '
status_str += '] {} '.format(path)
print(status_str, end='')
mp_image = mio.import_image(path)
mp_image.landmarks['bb'] = mio.import_landmark_file(
str(Path(mp_image.path.parent.parent / 'BoundingBoxes' / (mp_image.path.stem + '.pts')))
)
mp_image = mp_image.crop_to_landmarks_proportion(0.3, group='bb')
mp_image = mp_image.rescale_to_pointcloud(reference_shape, group='PTS')
mp_image = grey_to_rgb(mp_image)
# Padding to the same size
height, width = mp_image.pixels.shape[1:] # [C, H, W]
dy = max(int((image_shape[0] - height - 1) / 2), 0)
dx = max(int((image_shape[1] - width - 1) / 2), 0)
padded_image = np.random.rand(*image_shape).astype(np.float32)
padded_image[dy:(height + dy), dx:(width + dx), :] = mp_image.pixels.transpose(1, 2, 0)
padded_landmark = mp_image.landmarks['PTS'].points
padded_landmark[:, 0] += dy
padded_landmark[:, 1] += dx
features = tf.train.Features(
feature={
'train/image': tf.train.Feature(
bytes_list=tf.train.BytesList(value=[tf.compat.as_bytes(padded_image.tostring())])
),
'train/shape': tf.train.Feature(
float_list=tf.train.FloatList(value=padded_landmark.flatten())
)
}
)
ofs.write(tf.train.Example(features=features).SerializeToString())
if verbose:
print('')
# Sixth: test data
if Path(path_base / 'test.bin').exists():
pass
else:
with tf.io.TFRecordWriter(str(path_base / 'test.bin')) as ofs:
print('Preparing test data...')
counter = 0
for path in test_paths:
counter += 1
if verbose:
status = 10.0 * counter / len(test_paths)
status_str = '\rPreparing {:2.2f}%['.format(status * 10)
for i in range(int(status)):
status_str += '='
for i in range(int(status), 10):
status_str += ' '
status_str += '] {} '.format(path)
print(status_str, end='')
mp_image = mio.import_image(path)
mp_image.landmarks['bb'] = mio.import_landmark_file(
str(Path(mp_image.path.parent.parent / 'BoundingBoxes' / (mp_image.path.stem + '.pts')))
)
mp_image = mp_image.crop_to_landmarks_proportion(0.3, group='bb')
mp_bb = mp_image.landmarks['bb'].bounding_box()
mp_image.landmarks['init'] = align_shape_with_bounding_box(reference_shape, mp_bb)
mp_image = mp_image.rescale_to_pointcloud(reference_shape, group='init')
mp_image = grey_to_rgb(mp_image)
# Padding to the same size
height, width = mp_image.pixels.shape[1:] # [C, H, W]
dy = max(int((256 - height - 1) / 2), 0) # 200*(1+0.3*2)/sqrt(2) == 226.7
dx = max(int((256 - width - 1) / 2), 0) # 200*(1+0.3*2)/sqrt(2) == 226.7
padded_image = np.random.rand(256, 256, 3).astype(np.float32)
padded_image[dy:(height + dy), dx:(width + dx), :] = mp_image.pixels.transpose(1, 2, 0)
padded_landmark = mp_image.landmarks['PTS'].points
padded_landmark[:, 0] += dy
padded_landmark[:, 1] += dx
padded_init_landmark = mp_image.landmarks['init'].points
padded_init_landmark[:, 0] += dy
padded_init_landmark[:, 1] += dx
features = tf.train.Features(
feature={
'test/image': tf.train.Feature(
bytes_list=tf.train.BytesList(
value=[tf.compat.as_bytes(padded_image.tostring())])
),
'test/shape': tf.train.Feature(
float_list=tf.train.FloatList(value=padded_landmark.flatten())
),
'test/init': tf.train.Feature(
float_list=tf.train.FloatList(value=padded_init_landmark.flatten())
)
}
)
ofs.write(tf.train.Example(features=features).SerializeToString())
if verbose:
print('')
def lmark_resolver(path):
return mio.import_landmark_file(mio.data_path_to('takeo.pts'))
from numpy.testing import assert_allclose
import os
from pathlib import PosixPath, WindowsPath, Path
from mock import patch, PropertyMock, MagicMock
from pytest import raises
import menpo.io as mio
from menpo.io.utils import _norm_path
from menpo.image import Image
from menpo.io.output.pickle import pickle_paths_as_pure
builtins_str = '__builtin__' if sys.version_info[0] == 2 else 'builtins'
test_lg = mio.import_landmark_file(mio.data_path_to('lenna.ljson'),
group='LJSON')
nan_lg = test_lg.copy()
nan_lg.points[0, :] = np.nan
test_img = Image(np.random.random([100, 100]))
colour_test_img = Image(np.random.random([3, 100, 100]))
fake_path = '/tmp/test.fake'
@patch('menpo.io.output.base.landmark_types')
@patch('menpo.io.output.base.Path.exists')
@patch('menpo.io.output.base.Path.open')
def test_export_filepath_overwrite_exists(mock_open, exists, landmark_types):
exists.return_value = True
landmark_types.__contains__.return_value = True
mio.export_landmark_file(test_lg, fake_path, overwrite=True)
mock_open.assert_called_with('wb')
import numpy as np
from mock import patch, PropertyMock
from nose.tools import raises
import sys
import menpo.io as mio
from menpo.image import Image
builtins_str = '__builtin__' if sys.version_info[0] == 2 else 'builtins'
test_lg = mio.import_landmark_file(mio.data_path_to('breakingbad.pts'))
nan_lg = test_lg.copy()
nan_lg.lms.points[0, :] = np.nan
test_img = Image(np.random.random([100, 100]))
fake_path = '/tmp/test.fake'
@patch('menpo.io.output.base.landmark_types')
@patch('menpo.io.output.base.Path.exists')
@patch('menpo.io.output.base.Path.open')
def test_export_filepath_overwrite_exists(mock_open, exists, landmark_types):
exists.return_value = True
mio.export_landmark_file(test_lg, fake_path, overwrite=True)
mock_open.assert_called_once_with('wb')
landmark_types.__getitem__.assert_called_once_with('.fake')
export_function = landmark_types.__getitem__.return_value
export_function.assert_called_once()
@patch('menpo.io.output.base.landmark_types')
@patch('menpo.io.output.base.Path.exists')
import numpy as np
import sys
from numpy.testing import assert_allclose
import os
from pathlib import PosixPath, WindowsPath, Path
from mock import patch, PropertyMock, MagicMock
from nose.tools import raises
import menpo.io as mio
from menpo.io.utils import _norm_path
from menpo.image import Image
from menpo.io.output.pickle import pickle_paths_as_pure
builtins_str = '__builtin__' if sys.version_info[0] == 2 else 'builtins'
test_lg = mio.import_landmark_file(mio.data_path_to('lenna.ljson'))
nan_lg = test_lg.copy()
nan_lg.points[0, :] = np.nan
test_img = Image(np.random.random([100, 100]))
colour_test_img = Image(np.random.random([3, 100, 100]))
fake_path = '/tmp/test.fake'
@patch('menpo.io.output.base.landmark_types')
@patch('menpo.io.output.base.Path.exists')
@patch('menpo.io.output.base.Path.open')
def test_export_filepath_overwrite_exists(mock_open, exists, landmark_types):
exists.return_value = True
landmark_types.__contains__.return_value = True
mio.export_landmark_file(test_lg, fake_path, overwrite=True)
mock_open.assert_called_with('wb')
def load_images(paths, group=None, verbose=True):
"""Loads and rescales input images to the diagonal of the reference shape.
Args:
paths: a list of strings containing the data directories.
reference_shape: a numpy array [num_landmarks, 2]
group: landmark group containing the grounth truth landmarks.
verbose: boolean, print debugging info.
Returns:
images: a list of numpy arrays containing images.
shapes: a list of the ground truth landmarks.
reference_shape: a numpy array [num_landmarks, 2].
shape_gen: PCAModel, a shape generator.
"""
images = []
shapes = []
bbs = []
reference_shape = PointCloud(build_reference_shape(paths))
for path in paths:
if verbose:
print('Importing data from {}'.format(path))
for im in mio.import_images(path, verbose=verbose, as_generator=True):
group = group or im.landmarks[group]._group_label
bb_root = im.path.parent.relative_to(im.path.parent.parent.parent)
if 'set' not in str(bb_root):
bb_root = im.path.parent.relative_to(im.path.parent.parent)
im.landmarks['bb'] = mio.import_landmark_file(
str(Path('bbs') / bb_root / (im.path.stem + '.pts')))
im = im.crop_to_landmarks_proportion(0.3, group='bb')
im = im.rescale_to_pointcloud(reference_shape, group=group)
im = grey_to_rgb(im)
images.append(im.pixels.transpose(1, 2, 0))
shapes.append(im.landmarks[group].lms)
bbs.append(im.landmarks['bb'].lms)
train_dir = Path(FLAGS.train_dir)
mio.export_pickle(reference_shape.points,
train_dir / 'reference_shape.pkl',
overwrite=True)
print('created reference_shape.pkl using the {} group'.format(group))
pca_model = detect.create_generator(shapes, bbs)
# Pad images to max length
max_shape = np.max([im.shape for im in images], axis=0)
max_shape = [len(images)] + list(max_shape)
padded_images = np.random.rand(*max_shape).astype(np.float32)
print(padded_images.shape)
for i, im in enumerate(images):
height, width = im.shape[:2]
dy = max(int((max_shape[1] - height - 1) / 2), 0)
dx = max(int((max_shape[2] - width - 1) / 2), 0)
lms = shapes[i]
pts = lms.points
pts[:, 0] += dy
pts[:, 1] += dx
lms = lms.from_vector(pts)
padded_images[i, dy:(height + dy), dx:(width + dx)] = im
return padded_images, shapes, reference_shape.points, pca_model
import numpy as np
import sys
from numpy.testing import assert_allclose
import os
from pathlib import PosixPath, WindowsPath, Path
from mock import patch, PropertyMock, MagicMock
from nose.tools import raises
import menpo.io as mio
from menpo.io.utils import _norm_path
from menpo.image import Image
from menpo.io.output.pickle import pickle_paths_as_pure
builtins_str = '__builtin__' if sys.version_info[0] == 2 else 'builtins'
test_lg = mio.import_landmark_file(mio.data_path_to('breakingbad.pts'))
nan_lg = test_lg.copy()
nan_lg.lms.points[0, :] = np.nan
test_img = Image(np.random.random([100, 100]))
colour_test_img = Image(np.random.random([3, 100, 100]))
fake_path = '/tmp/test.fake'
@patch('menpo.io.output.base.landmark_types')
@patch('menpo.io.output.base.Path.exists')
@patch('menpo.io.output.base.Path.open')
def test_export_filepath_overwrite_exists(mock_open, exists, landmark_types):
exists.return_value = True
landmark_types.__contains__.return_value = True
mio.export_landmark_file(test_lg, fake_path, overwrite=True)
mock_open.assert_called_with('wb')
