def process_image(im_path, gt2d, coder, bbox_param, DRAW=False):
with tf.gfile.FastGFile(im_path, 'rb') as f:
image_data = f.read()
image = coder.decode_jpeg(image_data)
assert image.shape[2] == 3, \
'{} has {} channels.'.format(im_path, image.shape[2])
center = bbox_param[:2]
scale = bbox_param[2]
# estimate height..
# Using vis_threshold 0 for DT
vis = gt2d[:, 2] > 0.
image_scaled, scale_factors = resize_img(image, scale)
joints_scaled = np.copy(gt2d[:, :2])
joints_scaled[:, 0] *= scale_factors[0]
joints_scaled[:, 1] *= scale_factors[1]
center_scaled = np.round(center * scale_factors).astype(np.int)
# Make sure there is enough space to crop 300x300.
image_padded = np.pad(image_scaled, ((300, ), (300, ), (0, )), 'edge')
height, width = image_padded.shape[:2]
center_scaled += 300
joints_scaled += 300
# Crop 300x300 around the center.
margin = 150
start_pt = (center_scaled - margin).astype(int)
end_pt = (center_scaled + margin).astype(int)
end_pt[0] = min(end_pt[0], width)
end_pt[1] = min(end_pt[1], height)
image_scaled = image_padded[start_pt[1]:end_pt[1], start_pt[0]:end_pt[
0], :]
# Update others too.
joints_scaled[:, 0] -= start_pt[0]
joints_scaled[:, 1] -= start_pt[1]
center_scaled -= start_pt
height, width = image_scaled.shape[:2]
im_shape = [height, width]
# DRAW:
if DRAW:
import matplotlib.pyplot as plt
plt.ion()
plt.clf()
fig = plt.figure(1)
# ax = fig.add_subplot(131)
ax = fig.add_subplot(121)
image_with_skel = draw_skeleton(image, gt2d[:, :2], vis=vis)
ax.imshow(image_with_skel)
ax.axis('off')
ax.scatter(center[0], center[1], color='red')
# ax = fig.add_subplot(132)
ax = fig.add_subplot(122)
image_with_skel_scaled = draw_skeleton(
image_scaled, joints_scaled[:, :2], vis=vis)
ax.imshow(image_with_skel_scaled)
ax.scatter(center_scaled[0], center_scaled[1], color='red')
import ipdb
ipdb.set_trace()
# Encode image.
image_data_scaled = coder.encode_jpeg(image_scaled)
label = np.vstack([joints_scaled.T, vis])
return {
'image_data': image_data_scaled,
'image': image_scaled,
'image_shape': im_shape,
'label': label,
'center': center_scaled,
'scale_factors': scale_factors,
'start_pt': start_pt,
}
def visualize_img(img,
cam,
kp_pred,
vert,
renderer,
kp_gt=None,
text={},
rotated_view=False,
mesh_color='blue',
pad_vals=None,
no_text=False):
"""
Visualizes the image with the ground truth keypoints and
predicted keypoints on left and image with mesh on right.
Keypoints should be in normalized coordinates, not image coordinates.
Args:
img: Image.
cam (3x1): Camera parameters.
kp_gt: Ground truth keypoints.
kp_pred: Predicted keypoints.
vert: Vertices.
renderer: SMPL renderer.
text (dict): Optional information to include in the image.
rotated_view (bool): If True, also visualizes mesh from another angle.
if pad_vals (2,) is not None, removes those values from the image
(undo img pad to make square)
Returns:
Combined image.
"""
img_size = img.shape[0]
text.update({'sc': cam[0], 'tx': cam[1], 'ty': cam[2]})
if kp_gt is not None:
gt_vis = kp_gt[:, 2].astype(bool)
loss = np.sum((kp_gt[gt_vis, :2] - kp_pred[gt_vis])**2)
text['kpl'] = loss
# Undo pre-processing.
# Make sure img is [0-255]
input_img = ((img + 1) * 0.5) * 255.
rend_img = renderer(vert, cam=cam, img=input_img, color_name=mesh_color)
if not no_text:
rend_img = draw_text(rend_img, text)
# Draw skeletons
pred_joint = ((kp_pred + 1) * 0.5) * img_size
skel_img = draw_skeleton(input_img, pred_joint)
if kp_gt is not None:
gt_joint = ((kp_gt[:, :2] + 1) * 0.5) * img_size
skel_img = draw_skeleton(skel_img,
gt_joint,
draw_edges=False,
vis=gt_vis)
if pad_vals is not None:
skel_img = remove_pads(skel_img, pad_vals)
rend_img = remove_pads(rend_img, pad_vals)
if rotated_view:
rot_img = renderer.rotated(vert,
90,
cam=cam,
alpha=False,
color_name=mesh_color)
if pad_vals is not None:
rot_img = remove_pads(rot_img, pad_vals)
return skel_img / 255, rend_img / 255, rot_img / 255
else:
return skel_img / 255, rend_img / 255
def process_image(im_path,
gt2d,
coder,
bbox_param,
visualize=False,
vis_thresh=0.1,
img_size=224):
"""
Processes an image, producing 224x224 crop.
Args:
im_path (str).
gt2d (19x3).
coder (tf.ImageCoder).
bbox_param (3,): [cx, cy, scale].
visualize (bool).
vis_thresh (float).
img_size (int).
Returns:
dict: image_data_scaled, im_path, im_shape, kps, center,
scale, start_pt.
"""
with tf.gfile.FastGFile(im_path, 'rb') as f:
image_data = f.read()
image = coder.decode_jpeg(image_data)
assert image.shape[2] == 3
center = bbox_param[:2]
scale = bbox_param[2]
image_scaled, scale_factors = resize_img(image, scale)
vis = gt2d[:, 2] > vis_thresh
joints_scaled = np.copy(gt2d[:, :2])
joints_scaled[:, 0] *= scale_factors[0]
joints_scaled[:, 1] *= scale_factors[1]
center_scaled = np.round(center * scale_factors).astype(np.int)
# Make sure there is enough space to crop 224x224.
image_padded = np.pad(array=image_scaled,
pad_width=((img_size, ), (img_size, ), (0, )),
mode='edge')
height, width = image_padded.shape[:2]
center_scaled += img_size
joints_scaled += img_size
# Crop 224x224 around the center.
margin = img_size // 2
start_pt = (center_scaled - margin).astype(int)
end_pt = (center_scaled + margin).astype(int)
end_pt[0] = min(end_pt[0], width)
end_pt[1] = min(end_pt[1], height)
image_scaled = image_padded[start_pt[1]:end_pt[1],
start_pt[0]:end_pt[0], :]
# Update others too.
joints_scaled[:, 0] -= start_pt[0]
joints_scaled[:, 1] -= start_pt[1]
center_scaled -= start_pt
height, width = image_scaled.shape[:2]
im_shape = [height, width]
if visualize:
if gt2d is None:
image_with_skel = image
image_with_skel_scaled = image_scaled
else:
image_with_skel = draw_skeleton(image, gt2d[:, :2], vis=vis)
image_with_skel_scaled = draw_skeleton(image_scaled,
joints_scaled[:, :2],
vis=vis)
plt.ion()
plt.clf()
fig = plt.figure(1)
ax = fig.add_subplot(121)
ax.imshow(image_with_skel)
ax.axis('off')
ax.scatter(center[0], center[1], color='red')
ax = fig.add_subplot(122)
ax.imshow(image_with_skel_scaled)
ax.scatter(center_scaled[0], center_scaled[1], color='red')
plt.show()
plt.draw()
plt.pause(5e-6)
kps = np.vstack((joints_scaled.T, [vis]))
return {
'image_data_scaled': coder.encode_jpeg(image_scaled),
'im_path': im_path,
'im_shape': im_shape,
'kps': kps,
'center': center_scaled,
'scale': scale,
'start_pt': start_pt,
}
def process_image(
im_path,
gt2d,
cam,
coder,
pose=None,
shape=None,
gt3d=None,
vis=False,
):
# Read image.
with tf.gfile.FastGFile(im_path, 'rb') as f:
image_data = f.read()
image = coder.decode_jpeg(coder.png_to_jpeg(image_data))
assert image.shape[2] == 3
# Use gt2d to get the scale.
min_pt = np.min(gt2d, axis=0)
max_pt = np.max(gt2d, axis=0)
person_height = np.linalg.norm(max_pt - min_pt)
center = (min_pt + max_pt) / 2.
scale = 150. / person_height
image_scaled, scale_factors = resize_img(image, scale)
joints_scaled = np.copy(gt2d)
joints_scaled[:, 0] *= scale_factors[0]
joints_scaled[:, 1] *= scale_factors[1]
center_scaled = np.round(center * scale_factors).astype(np.int)
# scale camera:
cam_scaled = np.copy(cam)
# Flength
cam_scaled[0] *= scale
# px
cam_scaled[1] *= scale_factors[0]
# py
cam_scaled[2] *= scale_factors[1]
# Make sure there is enough space to crop 300x300.
image_padded = np.pad(image_scaled, ((300, ), (300, ), (0, )), 'edge')
height, width = image_padded.shape[:2]
center_scaled += 300
joints_scaled += 300
# Crop 300x300 around the center.
margin = 150
start_pt = (center_scaled - margin).astype(int)
end_pt = (center_scaled + margin).astype(int)
end_pt[0] = min(end_pt[0], width)
end_pt[1] = min(end_pt[1], height)
image_scaled = image_padded[start_pt[1]:end_pt[1],
start_pt[0]:end_pt[0], :]
# Update others too.
joints_scaled[:, 0] -= start_pt[0]
joints_scaled[:, 1] -= start_pt[1]
center_scaled -= start_pt
# Update principal point:
cam_scaled[1] += 300 - start_pt[0]
cam_scaled[2] += 300 - start_pt[1]
height, width = image_scaled.shape[:2]
im_shape = [height, width]
# Vis:
if vis:
import matplotlib.pyplot as plt
plt.ion()
plt.clf()
fig = plt.figure(1)
ax = fig.add_subplot(121)
image_with_skel = draw_skeleton(image, gt2d[:, :2])
ax.imshow(image_with_skel)
ax.axis('off')
ax.scatter(center[0], center[1], color='red')
ax = fig.add_subplot(122)
image_with_skel_scaled = draw_skeleton(image_scaled,
joints_scaled[:, :2])
ax.imshow(image_with_skel_scaled)
ax.scatter(center_scaled[0], center_scaled[1], color='red')
# Project it.
def project(X, c):
y = X[:, :2] / X[:, 2].reshape(-1, 1)
proj2d = c[0] * y + c[1:].reshape(1, -1)
return proj2d
proj2d = project(gt3d, cam_scaled)
ax.scatter(proj2d[:, 0], proj2d[:, 1], s=4)
ax.axis('off')
import ipdb
ipdb.set_trace()
# Encode image.
image_data_scaled = coder.encode_jpeg(image_scaled)
# Put things together.
label = np.vstack([joints_scaled.T, np.ones((1, len(COMMON_JOINT_IDS)))])
return {
'image_data': image_data_scaled,
'image': image_scaled,
'image_shape': im_shape,
'label': label,
'center': center_scaled,
'scale_factors': scale_factors,
'start_pt': start_pt,
'cam_scaled': cam_scaled,
}