def process_image(im_path, bbox_param):
"""
Processes an image, producing 224x224 crop.
Args:
im_path (str).
bbox_param (3,): [cx, cy, scale].
visualize (bool).
Returns:
dict: image, im_shape, center, scale_factors, start_pt.
"""
image = imread(im_path)
center = bbox_param[:2]
scale = bbox_param[2]
# Pre-process image to [-1, 1]
image = ((image / 255.) - 0.5) * 2
image_scaled, scale_factors = resize_img(image, scale)
center_scaled = np.round(center * scale_factors).astype(np.int)
# Make sure there is enough space to crop 224x224.
image_padded = np.pad(
array=image_scaled,
pad_width=((IMG_SIZE,), (IMG_SIZE,), (0,)),
mode='edge'
)
height, width = image_padded.shape[:2]
center_scaled += IMG_SIZE
# Crop 224x224 around the center.
margin = IMG_SIZE // 2
start_pt = (center_scaled - margin).astype(int)
end_pt = (center_scaled + margin).astype(int)
end_pt[0] = min(end_pt[0], width)
end_pt[1] = min(end_pt[1], height)
image_scaled = image_padded[start_pt[1]:end_pt[1],
start_pt[0]:end_pt[0], :]
center_scaled -= start_pt
height, width = image_scaled.shape[:2]
im_shape = [height, width]
return {
# return original too with info.
'image': image_scaled,
'im_path': im_path,
'im_shape': im_shape,
'center': center_scaled,
'scale': scale,
'start_pt': start_pt,
}
def visualize_mesh_og(cam,
vert,
renderer,
start_pt,
scale,
proc_img_shape,
im_path=None,
img=None,
deg=0,
mesh_color='blue',
max_img_size=300,
pad=50,
crop_cam=None,
bbox=None):
"""
Visualize mesh in original image space.
If you get out of memory error, make max_img_size smaller.
If crop_cam and bbox is not None,
crops the image and uses the crop_cam to render.
(See compute_video_bbox.py)
"""
if img is None:
img = imread(im_path)
# Pre-process image to [-1, 1] bc it expects this.
img = ((img / 255.) - 0.5) * 2
if bbox is not None:
assert (crop_cam is not None)
img = img[bbox[0]:bbox[1], bbox[2]:bbox[3]]
# For these, the cameras are already adjusted.
scale = 1.
start_pt = np.array([0, 0])
if np.max(img.shape[:2]) > max_img_size:
# if the image is too big it wont fit in gpu and nmr poops out.
scale_orig = max_img_size / float(np.max(img.shape[:2]))
img, _ = resize_img(img, scale_orig)
undo_scale = (1. / np.array(scale)) * scale_orig
else:
undo_scale = 1. / np.array(scale)
# NMR needs images to be square..
img, pad_vals = make_square(img)
img_size = np.max(img.shape[:2])
renderer.renderer.image_size = img_size
if bbox is not None:
return renderer.rotated(
verts=vert,
deg=deg,
cam=crop_cam,
color_name=mesh_color,
)
else:
# This is camera in crop image coord.
cam_crop = np.hstack(
[proc_img_shape[0] * cam[0] * 0.5, cam[1:] + (2. / cam[0]) * 0.5])
# This is camera in orig image coord
cam_orig = np.hstack([
cam_crop[0] * undo_scale,
cam_crop[1:] + (start_pt - proc_img_shape[0]) / cam_crop[0]
])
# This is the camera in normalized orig_image coord
new_cam = np.hstack([
cam_orig[0] * (2. / img_size),
cam_orig[1:] - (1 / ((2. / img_size) * cam_orig[0]))
])
new_cam = new_cam.astype(np.float32)
return renderer.rotated(
verts=vert,
deg=deg,
cam=new_cam,
color_name=mesh_color,
)
def visualize_img_orig(cam,
kp_pred,
vert,
renderer,
start_pt,
scale,
proc_img_shape,
im_path=None,
img=None,
rotated_view=False,
mesh_color='blue',
max_img_size=300,
no_text=False,
bbox=None,
crop_cam=None):
"""
Visualizes the image with the ground truth keypoints and predicted keypoints
in the original image space (squared).
If you get out of memory error, make max_img_size smaller.
Args:
must supply either the im_path or img
start_pt, scale, proc_img_shape are parameters used to preprocess the
image.
scale_result is how much to scale the current image
Returns:
Combined image.
"""
if img is None:
img = imread(im_path)
# Pre-process image to [-1, 1] bc it expects this.
img = ((img / 255.) - 0.5) * 2
if np.max(img.shape[:2]) > max_img_size:
# if the image is too big it wont fit in gpu and nmr poops out.
scale_orig = max_img_size / float(np.max(img.shape[:2]))
img, _ = resize_img(img, scale_orig)
undo_scale = (1. / np.array(scale)) * scale_orig
else:
undo_scale = 1. / np.array(scale)
if bbox is not None:
assert (crop_cam is not None)
img = img[bbox[0]:bbox[1], bbox[2]:bbox[3]]
# For these, the cameras are already adjusted.
start_pt = np.array([0, 0])
# NMR needs images to be square..
img, pad_vals = make_square(img)
img_size = np.max(img.shape[:2])
renderer.renderer.image_size = img_size
# Adjust kp_pred.
# This is in 224x224 cropped space.
pred_joint = ((kp_pred + 1) * 0.5) * proc_img_shape[0]
# This is in the original image.
pred_joint_orig = (pred_joint + start_pt - proc_img_shape[0]) * undo_scale
# in normalize coord of the original image:
kp_orig = 2 * (pred_joint_orig / img_size) - 1
if bbox is not None:
use_cam = crop_cam
else:
# This is camera in crop image coord.
cam_crop = np.hstack(
[proc_img_shape[0] * cam[0] * 0.5, cam[1:] + (2. / cam[0]) * 0.5])
# This is camera in orig image coord
cam_orig = np.hstack([
cam_crop[0] * undo_scale,
cam_crop[1:] + (start_pt - proc_img_shape[0]) / cam_crop[0]
])
# This is the camera in normalized orig_image coord
new_cam = np.hstack([
cam_orig[0] * (2. / img_size),
cam_orig[1:] - (1 / ((2. / img_size) * cam_orig[0]))
])
new_cam = new_cam.astype(np.float32)
use_cam = new_cam
# Call visualize_img with this camera:
rendered_orig = visualize_img(
img=img,
cam=use_cam,
kp_pred=kp_orig,
vert=vert,
renderer=renderer,
rotated_view=rotated_view,
mesh_color=mesh_color,
pad_vals=pad_vals,
no_text=no_text,
)
return rendered_orig
def process_image(im_path, gt2d, coder, bbox_param, DRAW=False):
with tf.gfile.FastGFile(im_path, 'rb') as f:
image_data = f.read()
image = coder.decode_jpeg(image_data)
assert image.shape[2] == 3, \
'{} has {} channels.'.format(im_path, image.shape[2])
center = bbox_param[:2]
scale = bbox_param[2]
# estimate height..
# Using vis_threshold 0 for DT
vis = gt2d[:, 2] > 0.
image_scaled, scale_factors = resize_img(image, scale)
joints_scaled = np.copy(gt2d[:, :2])
joints_scaled[:, 0] *= scale_factors[0]
joints_scaled[:, 1] *= scale_factors[1]
center_scaled = np.round(center * scale_factors).astype(np.int)
# Make sure there is enough space to crop 300x300.
image_padded = np.pad(image_scaled, ((300, ), (300, ), (0, )), 'edge')
height, width = image_padded.shape[:2]
center_scaled += 300
joints_scaled += 300
# Crop 300x300 around the center.
margin = 150
start_pt = (center_scaled - margin).astype(int)
end_pt = (center_scaled + margin).astype(int)
end_pt[0] = min(end_pt[0], width)
end_pt[1] = min(end_pt[1], height)
image_scaled = image_padded[start_pt[1]:end_pt[1], start_pt[0]:end_pt[
0], :]
# Update others too.
joints_scaled[:, 0] -= start_pt[0]
joints_scaled[:, 1] -= start_pt[1]
center_scaled -= start_pt
height, width = image_scaled.shape[:2]
im_shape = [height, width]
# DRAW:
if DRAW:
import matplotlib.pyplot as plt
plt.ion()
plt.clf()
fig = plt.figure(1)
# ax = fig.add_subplot(131)
ax = fig.add_subplot(121)
image_with_skel = draw_skeleton(image, gt2d[:, :2], vis=vis)
ax.imshow(image_with_skel)
ax.axis('off')
ax.scatter(center[0], center[1], color='red')
# ax = fig.add_subplot(132)
ax = fig.add_subplot(122)
image_with_skel_scaled = draw_skeleton(
image_scaled, joints_scaled[:, :2], vis=vis)
ax.imshow(image_with_skel_scaled)
ax.scatter(center_scaled[0], center_scaled[1], color='red')
import ipdb
ipdb.set_trace()
# Encode image.
image_data_scaled = coder.encode_jpeg(image_scaled)
label = np.vstack([joints_scaled.T, vis])
return {
'image_data': image_data_scaled,
'image': image_scaled,
'image_shape': im_shape,
'label': label,
'center': center_scaled,
'scale_factors': scale_factors,
'start_pt': start_pt,
}
def process_image(im_path,
gt2d,
coder,
bbox_param,
visualize=False,
vis_thresh=0.1,
img_size=224):
"""
Processes an image, producing 224x224 crop.
Args:
im_path (str).
gt2d (19x3).
coder (tf.ImageCoder).
bbox_param (3,): [cx, cy, scale].
visualize (bool).
vis_thresh (float).
img_size (int).
Returns:
dict: image_data_scaled, im_path, im_shape, kps, center,
scale, start_pt.
"""
with tf.gfile.FastGFile(im_path, 'rb') as f:
image_data = f.read()
image = coder.decode_jpeg(image_data)
assert image.shape[2] == 3
center = bbox_param[:2]
scale = bbox_param[2]
image_scaled, scale_factors = resize_img(image, scale)
vis = gt2d[:, 2] > vis_thresh
joints_scaled = np.copy(gt2d[:, :2])
joints_scaled[:, 0] *= scale_factors[0]
joints_scaled[:, 1] *= scale_factors[1]
center_scaled = np.round(center * scale_factors).astype(np.int)
# Make sure there is enough space to crop 224x224.
image_padded = np.pad(array=image_scaled,
pad_width=((img_size, ), (img_size, ), (0, )),
mode='edge')
height, width = image_padded.shape[:2]
center_scaled += img_size
joints_scaled += img_size
# Crop 224x224 around the center.
margin = img_size // 2
start_pt = (center_scaled - margin).astype(int)
end_pt = (center_scaled + margin).astype(int)
end_pt[0] = min(end_pt[0], width)
end_pt[1] = min(end_pt[1], height)
image_scaled = image_padded[start_pt[1]:end_pt[1],
start_pt[0]:end_pt[0], :]
# Update others too.
joints_scaled[:, 0] -= start_pt[0]
joints_scaled[:, 1] -= start_pt[1]
center_scaled -= start_pt
height, width = image_scaled.shape[:2]
im_shape = [height, width]
if visualize:
if gt2d is None:
image_with_skel = image
image_with_skel_scaled = image_scaled
else:
image_with_skel = draw_skeleton(image, gt2d[:, :2], vis=vis)
image_with_skel_scaled = draw_skeleton(image_scaled,
joints_scaled[:, :2],
vis=vis)
plt.ion()
plt.clf()
fig = plt.figure(1)
ax = fig.add_subplot(121)
ax.imshow(image_with_skel)
ax.axis('off')
ax.scatter(center[0], center[1], color='red')
ax = fig.add_subplot(122)
ax.imshow(image_with_skel_scaled)
ax.scatter(center_scaled[0], center_scaled[1], color='red')
plt.show()
plt.draw()
plt.pause(5e-6)
kps = np.vstack((joints_scaled.T, [vis]))
return {
'image_data_scaled': coder.encode_jpeg(image_scaled),
'im_path': im_path,
'im_shape': im_shape,
'kps': kps,
'center': center_scaled,
'scale': scale,
'start_pt': start_pt,
}
def process_image(
im_path,
gt2d,
cam,
coder,
pose=None,
shape=None,
gt3d=None,
vis=False,
):
# Read image.
with tf.gfile.FastGFile(im_path, 'rb') as f:
image_data = f.read()
image = coder.decode_jpeg(coder.png_to_jpeg(image_data))
assert image.shape[2] == 3
# Use gt2d to get the scale.
min_pt = np.min(gt2d, axis=0)
max_pt = np.max(gt2d, axis=0)
person_height = np.linalg.norm(max_pt - min_pt)
center = (min_pt + max_pt) / 2.
scale = 150. / person_height
image_scaled, scale_factors = resize_img(image, scale)
joints_scaled = np.copy(gt2d)
joints_scaled[:, 0] *= scale_factors[0]
joints_scaled[:, 1] *= scale_factors[1]
center_scaled = np.round(center * scale_factors).astype(np.int)
# scale camera:
cam_scaled = np.copy(cam)
# Flength
cam_scaled[0] *= scale
# px
cam_scaled[1] *= scale_factors[0]
# py
cam_scaled[2] *= scale_factors[1]
# Make sure there is enough space to crop 300x300.
image_padded = np.pad(image_scaled, ((300, ), (300, ), (0, )), 'edge')
height, width = image_padded.shape[:2]
center_scaled += 300
joints_scaled += 300
# Crop 300x300 around the center.
margin = 150
start_pt = (center_scaled - margin).astype(int)
end_pt = (center_scaled + margin).astype(int)
end_pt[0] = min(end_pt[0], width)
end_pt[1] = min(end_pt[1], height)
image_scaled = image_padded[start_pt[1]:end_pt[1],
start_pt[0]:end_pt[0], :]
# Update others too.
joints_scaled[:, 0] -= start_pt[0]
joints_scaled[:, 1] -= start_pt[1]
center_scaled -= start_pt
# Update principal point:
cam_scaled[1] += 300 - start_pt[0]
cam_scaled[2] += 300 - start_pt[1]
height, width = image_scaled.shape[:2]
im_shape = [height, width]
# Vis:
if vis:
import matplotlib.pyplot as plt
plt.ion()
plt.clf()
fig = plt.figure(1)
ax = fig.add_subplot(121)
image_with_skel = draw_skeleton(image, gt2d[:, :2])
ax.imshow(image_with_skel)
ax.axis('off')
ax.scatter(center[0], center[1], color='red')
ax = fig.add_subplot(122)
image_with_skel_scaled = draw_skeleton(image_scaled,
joints_scaled[:, :2])
ax.imshow(image_with_skel_scaled)
ax.scatter(center_scaled[0], center_scaled[1], color='red')
# Project it.
def project(X, c):
y = X[:, :2] / X[:, 2].reshape(-1, 1)
proj2d = c[0] * y + c[1:].reshape(1, -1)
return proj2d
proj2d = project(gt3d, cam_scaled)
ax.scatter(proj2d[:, 0], proj2d[:, 1], s=4)
ax.axis('off')
import ipdb
ipdb.set_trace()
# Encode image.
image_data_scaled = coder.encode_jpeg(image_scaled)
# Put things together.
label = np.vstack([joints_scaled.T, np.ones((1, len(COMMON_JOINT_IDS)))])
return {
'image_data': image_data_scaled,
'image': image_scaled,
'image_shape': im_shape,
'label': label,
'center': center_scaled,
'scale_factors': scale_factors,
'start_pt': start_pt,
'cam_scaled': cam_scaled,
}