def visualize_3d2(img_t, proc_param_t, joints_t, verts_t, cam_t, joints3d_t,
img_q, proc_param_q, joints_q, verts_q, cam_q, joints3d_q):
"""
Renders the result in original image coordinate frame.
"""
import matplotlib.pyplot as plt
fig = plt.figure()
## Pose 1
cam_for_render_t, vert_shifted_t, joints_orig_t = vis_util.get_original(
proc_param_t, verts_t, cam_t, joints_t, img_size=img_t.shape[:2])
# Render results
ax_3d_t = fig.add_subplot(221, projection='3d')
ax_3d_t = vis_util.draw_skeleton_3d(joints3d_t, ax_3d_t)
ax_img_t = fig.add_subplot(222)
skel_img_t = vis_util.draw_skeleton(img_t, joints_orig_t)
ax_img_t.imshow(skel_img_t)
## Pose 2
cam_for_render_q, vert_shifted_q, joints_orig_q = vis_util.get_original(
proc_param_q, verts_q, cam_q, joints_q, img_size=img_q.shape[:2])
# Render results
ax_3d_q = fig.add_subplot(223, projection='3d')
ax_3d_q = vis_util.draw_skeleton_3d(joints3d_q, ax_3d_q)
ax_img_q = fig.add_subplot(224)
skel_img_q = vis_util.draw_skeleton(img_q, joints_orig_q)
ax_img_q.imshow(skel_img_q)
plt.draw()
plt.show()
def visualize(img, proc_param, joints, verts, cam, img_path):
"""
Renders the result in original image coordinate frame.
"""
cam_for_render, vert_shifted, joints_orig = vis_util.get_original(
proc_param, verts, cam, joints, img_size=img.shape[:2])
centered_cam = (0.9, 0, 0) # used to see fully-visible views of person
cam_for_render2, vert_shifted2, joints_orig2 = vis_util.get_original(
proc_param, verts, centered_cam, joints, img_size=img.shape[:2])
# Render results
skel_img = vis_util.draw_skeleton(img, joints)
f = 5.
tz = f / cam[0]
cam_for_render = 0.5 * 224 * np.array([f, 1, 1])
cam_t = np.array([cam[1], cam[2], tz])
rend_img_overlay = renderer(verts + cam_t, cam=cam_for_render, img=img)
#rend_img_overlay = renderer(
# vert_shifted, cam=cam_for_render, img=img, do_alpha=True)
rend_img = renderer(verts + cam_t,
cam=cam_for_render,
img_size=img.shape[:2])
cam_t2 = np.array([cam[1], cam[2], tz * 2])
rend_img_vp1 = renderer.rotated(verts + cam_t2,
60,
cam=cam_for_render,
img_size=img.shape[:2])
rend_img_vp2 = renderer.rotated(verts + cam_t2,
-60,
cam=cam_for_render,
img_size=img.shape[:2])
import matplotlib
matplotlib.use('agg')
import matplotlib.pyplot as plt
# plt.ion()
plt.figure(1)
plt.clf()
plt.subplot(131)
plt.imshow(img)
plt.title('input')
plt.axis('off')
plt.subplot(132)
plt.imshow(rend_img_overlay)
plt.title('3D Mesh overlay')
plt.axis('off')
plt.subplot(133)
plt.imshow(rend_img)
plt.title('3D mesh')
plt.axis('off')
plt.draw()
plt.show()
plt.savefig(img_path[:-4] + '_preds' + '.png')
def visualize_3d(img, proc_param, joints, verts, cam, joints3d):
"""
Renders the result in original image coordinate frame.
"""
import matplotlib.pyplot as plt
cam_for_render, vert_shifted, joints_orig = vis_util.get_original(
proc_param, verts, cam, joints, img_size=img.shape[:2])
# Render results
fig = plt.figure()
ax = fig.add_subplot(121, projection='3d')
print("Joints shape 3d:" + str(joints3d.shape))
ax = vis_util.draw_skeleton_3d(joints3d, ax)
#plt = vis_util.draw_skeleton_3d(img, joints_orig, plt)
ax1 = fig.add_subplot(122)
skel_img = vis_util.draw_skeleton(img, joints_orig)
ax1.imshow(skel_img)
# plt.ion()
plt.title('diff vp')
plt.axis('off')
plt.draw()
plt.show()
def main(vid_path, json_path=None):
sess = tf.Session()
model = RunModel(config, sess=sess)
# Video capture
cap = cv2.VideoCapture(vid_path)
# verts_anim = []
while True:
ret, frame = cap.read()
input_img, proc_param, img = preprocess_image(frame, json_path)
# Add batch dimension: 1 x D x D x 3
input_img = np.expand_dims(input_img, 0)
# Theta is the 85D vector holding [camera, pose, shape]
# where camera is 3D [s, tx, ty]
# pose is 72D vector holding the rotation of 24 joints of SMPL in axis angle format
# shape is 10D shape coefficients of SMPL
joints, verts, cams, joints3d, theta = model.predict(input_img,
get_theta=True)
# verts_anim.append(verts)
# print (verts)
cam_for_render, vert_shifted, joints_orig = vis_util.get_original(
proc_param, verts, cams[0], joints[0], img_size=img.shape[:2])
rend_img_overlay = renderer(vert_shifted,
cam=cam_for_render,
img=img,
do_alpha=True)
cv2.imshow('processed', rend_img_overlay)
cv2.imshow('processed1', frame)
def visualize(img, proc_param, joints, verts, cam, image_name=None):
"""
Renders the result in original image coordinate frame.
"""
cam_for_render, vert_shifted, joints_orig = vis_util.get_original(
proc_param, verts, cam, joints, img_size=img.shape[:2])
# Render results
skel_img = vis_util.draw_skeleton(img, joints_orig)
# rend_img_overlay = renderer(
# vert_shifted, cam=cam_for_render, img=img, do_alpha=True)
# rend_img = renderer(
# vert_shifted, cam=cam_for_render, img_size=img.shape[:2])
# rend_img_vp1 = renderer.rotated(
# vert_shifted, 60, cam=cam_for_render, img_size=img.shape[:2])
# rend_img_vp2 = renderer.rotated(
# vert_shifted, -60, cam=cam_for_render, img_size=img.shape[:2])
cv2.namedWindow('input' + image_name, cv2.WINDOW_NORMAL)
cv2.namedWindow('joint projection' + image_name, cv2.WINDOW_NORMAL)
cv2.imshow('input' + image_name, img)
cv2.imshow('joint projection' + image_name, skel_img)
# cv2.imshow('3D Mesh overlay',rend_img_overlay)
# cv2.imshow('3D mesh',rend_img)
# cv2.imshow('diff vp',rend_img_vp1)
# cv2.imshow('diff vp 2',rend_img_vp2)
cv2.waitKey(25)
"""
def visualize(img, proc_param, joints, verts, cam):
"""
Renders the result in original image coordinate frame.
"""
cam_for_render, vert_shifted, joints_orig = vis_util.get_original(
proc_param, verts, cam, joints, img_size=img.shape[:2])
# Render results
skel_img = vis_util.draw_skeleton(img, joints_orig)
rend_img_overlay = renderer(vert_shifted,
cam=cam_for_render,
img=img,
do_alpha=True)
rend_img = renderer(vert_shifted,
cam=cam_for_render,
img_size=img.shape[:2])
rend_img_vp1 = renderer.rotated(vert_shifted,
60,
cam=cam_for_render,
img_size=img.shape[:2])
rend_img_vp2 = renderer.rotated(vert_shifted,
-60,
cam=cam_for_render,
img_size=img.shape[:2])
'''
def visualize(img, proc_param, joints_crop, verts_crop, cam_crop):
# render using opendr
cam_render, verts_render, joints_render = vis_util.get_original(
proc_param, verts_crop, cam_crop, joints_crop, img_size=img.shape[:2])
skel_img = vis_util.draw_skeleton(img, joints_render)
rend_img_overlay = renderer(verts_render,
cam=cam_render,
img=img,
do_alpha=True)
rend_img = renderer(verts_render, cam=cam_render, img_size=img.shape[:2])
rend_img_vp1 = renderer.rotated(verts_render,
60,
cam=cam_render,
img_size=img.shape[:2])
rend_img_vp2 = renderer.rotated(verts_render,
-60,
cam=cam_render,
img_size=img.shape[:2])
# draw verts projected on original image
verts2d = project_3d_to_original2d_v2(cam_crop, verts_crop, proc_param)
verts_z = verts_crop[:, 2]
verts2d_img = draw_2d_verts(np.zeros(img.shape, dtype=np.uint8), verts2d,
verts_z)
verts2d_img_overlay = draw_2d_verts(img, verts2d, verts_z)
verts2d_rend_overlay = draw_2d_verts(rend_img, verts2d, verts_z)
row1 = np.hstack([img, skel_img, rend_img_overlay[..., :3]])
row2 = np.hstack([rend_img, rend_img_vp1[..., :3], rend_img_vp2[..., :3]])
row3 = np.hstack([verts2d_img, verts2d_img_overlay, verts2d_rend_overlay])
vis = np.vstack([row1, row2, row3])
return vis
def rec_human(pipe_img_2, pipe_center, pipe_scale):
config = flags.FLAGS
config(sys.argv)
config.load_path = src.config.PRETRAINED_MODEL
config.batch_size = 1
sess = tf.Session()
model = RunModel(config, sess=sess)
print(config.smpl_face_path)
rec_human_count = 0
rec_human_time = time.time()
while True:
img = pipe_img_2.recv()
center = pipe_center.recv()
scale = pipe_scale.recv()
input_img, proc_param = img_util.scale_and_crop(
img, scale, center, config.img_size)
input_img = 2 * ((input_img / 255.) - 0.5)
input_img = np.expand_dims(input_img, 0)
joints, verts, cams, joints3d, theta = model.predict(input_img,
get_theta=True)
cam_for_render, vert_shifted, joints_orig = vis_util.get_original(
proc_param, verts[0], cams[0], joints[0], img_size=img.shape[:2])
#print(cam_for_render.shape)
rec_human_count = rec_human_count + 1
if rec_human_count == 100:
print('rec FPS:', 1.0 / ((time.time() - rec_human_time) / 100.0))
rec_human_count = 0
rec_human_time = time.time()
def visualize(img, proc_param, joints, verts, cam):
"""
Renders the result in original image coordinate frame.
"""
cam_for_render, vert_shifted, joints_orig = vis_util.get_original(
proc_param, verts, cam, joints, img_size=img.shape[:2])
# Render results
skel_img = vis_util.draw_skeleton(img, joints_orig)
rend_img_overlay = renderer(vert_shifted,
cam=cam_for_render,
img=img,
do_alpha=True)
rend_img = renderer(vert_shifted,
cam=cam_for_render,
img_size=img.shape[:2])
rend_img_vp1 = renderer.rotated(vert_shifted,
60,
cam=cam_for_render,
img_size=img.shape[:2])
rend_img_vp2 = renderer.rotated(vert_shifted,
-60,
cam=cam_for_render,
img_size=img.shape[:2])
import matplotlib.pyplot as plt
# plt.ion()
plt.figure(1)
plt.clf()
plt.imshow(rend_img_overlay)
plt.title('3D Mesh overlay')
plt.axis('off')
plt.draw()
plt.savefig('./HMR_moi_7.png')
plt.show()
def rerenders(img_path, proc_param, joints, verts, cam, folder_name):
try:
import hmr.src.config
from hmr.src.util import renderer as vis_util
except:
pass
config = flags.FLAGS
config(sys.argv)
# Using pre-trained model, change this to use your own.
config.load_path = hmr.src.config.PRETRAINED_MODEL
config.batch_size = 1
renderer = vis_util.SMPLRenderer(face_path=config.smpl_face_path)
"""
Renders the result in original image coordinate frame.
"""
img = io.imread(img_path)
cam_for_render, vert_shifted, joints_orig = vis_util.get_original(
proc_param, verts, cam, joints, img_size=img.shape[:2])
# Render results
rend_img_overlay = renderer(
vert_shifted, cam=cam_for_render, img=img, do_alpha=True)
rend_img_overlay = cv2.cvtColor(rend_img_overlay, cv2.COLOR_BGR2RGB)
# print("hhhhhhhhhhhhhhhhh/home/ankur/GUI_project/frames/FRAMES_HMR/" + folder_name + img_path.split('/')[-1][:-4]+".png")
cv2.imwrite("/home/ankur/GUI_project/frames/FRAMES_HMR/" + folder_name + "/" + img_path.split('/')[-1][:-4]+".png",rend_img_overlay)
def visualize(img, proc_param, joints, verts, cam):
"""
Renders the result in original image coordinate frame.
"""
cam_for_render, vert_shifted, joints_orig = vis_util.get_original(
proc_param, verts, cam, joints, img_size=img.shape[:2])
# Render results
skel_img = vis_util.draw_skeleton(img, joints_orig)
rend_img_overlay = renderer(vert_shifted,
cam=cam_for_render,
img=img,
do_alpha=True)
rend_img = renderer(vert_shifted,
cam=cam_for_render,
img_size=img.shape[:2])
rend_img_vp1 = renderer.rotated(vert_shifted,
60,
cam=cam_for_render,
img_size=img.shape[:2])
rend_img_vp2 = renderer.rotated(vert_shifted,
-60,
cam=cam_for_render,
img_size=img.shape[:2])
# plt.ion()
# plt.figure(1)
# plt.clf()
# plt.subplot(231)
# plt.imshow(img)
# plt.title('input')
# plt.axis('off')
# plt.subplot(232)
# plt.imshow(skel_img)
out.write(skel_img)
def visualize_and_save(img, proc_param, joints, verts, cam, save_path,
frame_num):
"""
Renders the result in original image coordinate frame.
"""
cam_for_render, vert_shifted, joints_orig = vis_util.get_original(
proc_param, verts, cam, joints, img_size=img.shape[:2])
# Render results
skel_img = vis_util.draw_skeleton(img, joints_orig)
rend_img_overlay = renderer(vert_shifted,
cam=cam_for_render,
img=img,
do_alpha=True)
rend_img = renderer(vert_shifted,
cam=cam_for_render,
img_size=img.shape[:2])
rend_img_vp1 = renderer.rotated(vert_shifted,
60,
cam=cam_for_render,
img_size=img.shape[:2])
rend_img_vp2 = renderer.rotated(vert_shifted,
-60,
cam=cam_for_render,
img_size=img.shape[:2])
combined_img = np.concatenate(
(rend_img_overlay, rend_img_vp1, rend_img_vp2), axis=0)
cv2.imshow('frame', combined_img)
if save_path is not None:
cv2.imwrite(save_path + str(frame_num) + "hmr_result.png",
combined_img)
def p3d(img, joints3d, joints, verts, cam, proc_param, filename):
#(0 - r ankle,
# 1 - r knee,
# 2 - r hip,
# 3 - l hip,
# 4 - l knee,
# 5 - l ankle,
# 6 - pelvis,
# 7 - thorax,
# 8 - upper neck,
# 9 - head top,
# 10 - r wrist,
# 10 - r wrist,
# 12 - r shoulder,
# 13 - l shoulder,
# 14 - l elbow,
# 15 - l wrist)
limb_parents = [1, 2, 12, 12, 3, 4, 7, 8, 12, 12, 9, 10, 13, 13, 13]
import matplotlib.pyplot as plt
from mpl_toolkits import mplot3d
cam_for_render, vert_shifted, joints_orig = vis_util.get_original(
proc_param, verts, cam, joints, img_size=img.shape[:2])
img_size = proc_param['img_size']
undo_scale = 1. / np.array(proc_param['scale'])
cam_s = cam[0]
cam_pos = cam[1:]
flength = 500.
tz = flength / (0.5 * img_size * cam_s)
pp_orig = cam_for_render[1:] / (img_size * undo_scale)
print(pp_orig)
trans = np.hstack([pp_orig, tz])
joints3d = joints3d[0, :14, :]
joints3d = joints3d + trans
print(joints3d)
# plt.ion()
plt.figure(1)
plt.clf()
ax = plt.axes(projection='3d')
ax.set_xlim(0, 2)
ax.set_ylim(0, 2)
ax.set_zlim(0, 10)
ax.view_init(elev=-70, azim=-90)
ax.scatter3D(joints3d[:, 0], joints3d[:, 1], joints3d[:, 2], 'gray')
for i in range(joints3d.shape[0]):
x_pair = [joints3d[i, 0], joints3d[limb_parents[i], 0]]
y_pair = [joints3d[i, 1], joints3d[limb_parents[i], 1]]
z_pair = [joints3d[i, 2], joints3d[limb_parents[i], 2]]
ax.plot(x_pair, y_pair, zs=z_pair, linewidth=3)
ax.axis('off')
#plt.show()
plt.savefig(os.path.join("output", filename))
def visualize(img_path, img, proc_param, joints, verts, cam):
"""
Renders the result in original image coordinate frame.
"""
cam_for_render, vert_shifted, joints_orig = vis_util.get_original(
proc_param, verts, cam, joints, img_size=img.shape[:2])
# Render results
skel_img = vis_util.draw_skeleton(img, joints_orig)
rend_img_overlay = renderer(vert_shifted,
cam=cam_for_render,
img=img,
do_alpha=True)
rend_img = renderer(vert_shifted,
cam=cam_for_render,
img_size=img.shape[:2])
rend_img_vp1 = renderer.rotated(vert_shifted,
60,
cam=cam_for_render,
img_size=img.shape[:2])
rend_img_vp2 = renderer.rotated(vert_shifted,
-60,
cam=cam_for_render,
img_size=img.shape[:2])
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
# plt.ion()
plt.figure(1)
plt.clf()
plt.subplot(231)
plt.imshow(img)
plt.title('input')
plt.axis('off')
plt.subplot(232)
plt.imshow(skel_img)
plt.title('joint projection')
plt.axis('off')
plt.subplot(233)
plt.imshow(rend_img_overlay)
plt.title('3D Mesh overlay')
plt.axis('off')
plt.subplot(234)
plt.imshow(rend_img)
plt.title('3D mesh')
plt.axis('off')
plt.subplot(235)
plt.imshow(rend_img_vp1)
plt.title('diff vp')
plt.axis('off')
plt.subplot(236)
plt.imshow(rend_img_vp2)
plt.title('diff vp')
plt.axis('off')
plt.draw()
plt.savefig("hmr/output/images/" +
os.path.splitext(os.path.basename(img_path))[0] + ".png")
def save_mesh(img, img_path, proc_param, joints, verts, cam):
cam_for_render, vert_3d, joints_orig = vis_util.get_original(
proc_param, verts, cam, joints, img_size=img.shape[:2])
cam_for_render, vert_shifted = cam, verts
#print(proc_param)
#print(vert_shifted)
camera = np.reshape(cam_for_render, [1, 3])
w, h, _ = img.shape
imgsize = max(w, h)
# project to 2D
vert_2d = verts[:, :2] + camera[:, 1:]
vert_2d = vert_2d * camera[0, 0]
img_copy = img.copy()
face_path = './src/tf_smpl/smpl_faces.npy'
faces = np.load(face_path)
obj_mesh_name = 'test.obj'
foreground_index_2d = np.zeros((w, h)) + 99999
foreground_value_2d = np.zeros((w, h)) + 99999
with open(obj_mesh_name, 'w') as fp:
# Decide Forground
for i in range(vert_2d.shape[0]):
v2 = vert_2d[i, :]
v3 = vert_3d[i, :]
z = v3[2]
x = int(round((v2[1] + 1) * 0.5 * imgsize))
y = int(round((v2[0] + 1) * 0.5 * imgsize))
if w < h:
x = int(round(x - h / 2 + w / 2))
else:
y = int(round(y - w / 2 + h / 2))
x = max(0, min(x, w - 1))
y = max(0, min(y, h - 1))
if z < foreground_value_2d[x, y]:
foreground_index_2d[x, y] = i
foreground_value_2d[x, y] = z
# Draw Color
for i in range(vert_2d.shape[0]):
v2 = vert_2d[i, :]
v3 = vert_3d[i, :]
z = v3[2]
x = int(round((v2[1] + 1) * 0.5 * imgsize))
y = int(round((v2[0] + 1) * 0.5 * imgsize))
if w < h:
x = int(round(x - h / 2 + w / 2))
else:
y = int(round(y - w / 2 + h / 2))
x = max(0, min(x, w - 1))
y = max(0, min(y, h - 1))
if i == foreground_index_2d[x, y]:
c = img[x, y, :] / 255.0
img_copy[x, y, :] = 0
else:
c = [1, 1, 1]
fp.write('v %f %f %f %f %f %f\n' %
(v3[0], v3[1], v3[2], c[0], c[1], c[2]))
for f in faces: # Faces are 1-based, not 0-based in obj files
fp.write('f %d %d %d\n' % (f[0] + 1, f[1] + 1, f[2] + 1))
img_copy = Image.fromarray(img_copy, 'RGB')
img_copy.save('input.png')
def save(img, proc_param, joints, verts, cam, i):
"""
Renders the result in original image coordinate frame.
"""
cam_for_render, vert_shifted, joints_orig = vis_util.get_original(
proc_param, verts, cam, joints, img_size=img.shape[:2])
# Render results
skel_img = vis_util.draw_skeleton(img, joints_orig)
rend_img_overlay = renderer(vert_shifted,
cam=cam_for_render,
img=img,
do_alpha=True)
rend_img = renderer(vert_shifted,
cam=cam_for_render,
img_size=img.shape[:2])
rend_img_vp1 = renderer.rotated(vert_shifted,
60,
cam=cam_for_render,
img_size=img.shape[:2])
rend_img_vp2 = renderer.rotated(vert_shifted,
-60,
cam=cam_for_render,
img_size=img.shape[:2])
import matplotlib.pyplot as plt
# plt.ion()
plt.figure(i)
plt.clf()
plt.subplot(231)
plt.imshow(img)
plt.title('input')
plt.axis('off')
plt.subplot(232)
plt.imshow(skel_img)
plt.title('joint projection')
plt.axis('off')
plt.subplot(233)
plt.imshow(rend_img_overlay)
plt.title('3D Mesh overlay')
plt.axis('off')
plt.subplot(234)
plt.imshow(rend_img)
plt.title('3D mesh')
plt.axis('off')
plt.subplot(235)
plt.imshow(rend_img_vp1)
plt.title('diff vp')
plt.axis('off')
plt.subplot(236)
plt.imshow(rend_img_vp2)
plt.title('diff vp')
plt.axis('off')
plt.draw()
#plt.show()
name = str(i) + ".png"
print(name)
plt.savefig(name)
def save_mesh(img, img_path, split, proc_param, joints, verts, cam, faces):
cam_for_render, vert_3d, joints_orig = vis_util.get_original(
proc_param, verts, cam, joints, img_size=img.shape[:2])
cam_for_render, vert_shifted = cam, verts
#print(proc_param)
#print(vert_shifted)
camera = np.reshape(cam_for_render, [1,3])
w, h, _ = img.shape
imgsize = max(w,h)
# project to 2D
vert_2d = verts[:, :2] + camera[:, 1:]
vert_2d = vert_2d * camera[0,0]
img_copy = img.copy()
mask_name = './3DMarket_hmr_mask/%s/%s/%s.png'%( split, os.path.basename(os.path.dirname(img_path)), os.path.basename(img_path) )
store_dir = os.path.dirname(mask_name)
if not os.path.exists(os.path.dirname(store_dir)):
os.mkdir(os.path.dirname(store_dir))
if not os.path.exists(store_dir):
os.mkdir(store_dir)
foreground_index_2d = np.zeros((w,h))+99999
foreground_value_2d = np.zeros((w,h))+99999
background = np.zeros((w,h))
mask = np.zeros((w,h))
index = 6891
w, h, _ = img.shape
imgsize = max(w,h)
# Decide Forground
for i in range(vert_2d.shape[0]):
v2 = vert_2d[i,:]
v3 = vert_3d[i,:]
z = v3[2]
x = int(round( (v2[1]+1)*0.5*imgsize ))
y = int(round( (v2[0]+1)*0.5*imgsize ))
if w<h:
x = int(round(x -h/2 + w/2))
else:
y = int(round(y - w/2 + h/2))
x = max(0, min(x, w-1))
y = max(0, min(y, h-1))
mask[x, y] = 1
if z < foreground_value_2d[x,y]:
foreground_index_2d[x,y] = i
foreground_value_2d[x,y] = z
# check the hole
for t in range(3):
for i in range(1,w-1):
for j in range(1,h-1):
if mask[i,j] == 1:
continue
sum = mask[i-1,j-1] + mask[i,j-1] + mask[i-1,j] + mask[i-1,j+1] \
+mask[i+1,j+1] + mask[i,j+1] + mask[i+1,j] + mask[i+1,j-1]
if sum >= 5:
mask[i, j] = 1
#img_masked = img*mask.reshape(w,h,1)
#img_masked = Image.fromarray(np.uint8(img_masked))
#img_masked.save(mask_name+'.jpg')
mask = Image.fromarray(np.uint8(mask*255))
mask.save(mask_name)
def get_render(img, proc_param, joints, verts, cam):
"""
Renders the result in original image coordinate frame.
"""
cam_for_render, vert_shifted, joints_orig = rnd.get_original(
proc_param, verts, cam, joints, img_size=img.shape[:2])
rend_img_overlay = renderer(
vert_shifted, cam=cam_for_render, img=img, do_alpha=True)
return rend_img_overlay
def visualize(img, proc_param, renderer, joints, verts, cam):
"""
Renders the result in original image coordinate frame.
"""
cam_for_render, vert_shifted, joints_orig = vis_util.get_original(
proc_param, verts, cam, joints, img_size=img.shape[:2])
# Render results
skel_img = vis_util.draw_skeleton(img, joints_orig)
rend_img_overlay = renderer(vert_shifted,
cam=cam_for_render,
img=img,
do_alpha=True)
rend_img = renderer(vert_shifted,
cam=cam_for_render,
img_size=img.shape[:2])
rend_img_vp1 = renderer.rotated(vert_shifted,
60,
cam=cam_for_render,
img_size=img.shape[:2])
rend_img_vp2 = renderer.rotated(vert_shifted,
-60,
cam=cam_for_render,
img_size=img.shape[:2])
# plt.ion()
fig = plt.figure(1)
plt.clf()
plt.subplot(231)
plt.imshow(img)
plt.title('input')
plt.axis('off')
plt.subplot(232)
plt.imshow(skel_img)
plt.title('joint projection')
plt.axis('off')
plt.subplot(233)
plt.imshow(rend_img_overlay)
plt.title('3D Mesh overlay')
plt.axis('off')
plt.subplot(234)
plt.imshow(rend_img)
plt.title('3D mesh')
plt.axis('off')
plt.subplot(235)
plt.imshow(rend_img_vp1)
plt.title('diff vp')
plt.axis('off')
plt.subplot(236)
plt.imshow(rend_img_vp2)
plt.title('diff vp')
plt.axis('off')
plt.draw()
plt.show()
# import ipdb
# ipdb.set_trace()
return fig
def visualize_3d3(img_t, proc_param_t, joints_t, verts_t, cam_t, joints3d_t,
img_q, proc_param_q, joints_q, verts_q, cam_q, joints3d_q):
"""
Renders the result in original image coordinate frame.
"""
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
gs = gridspec.GridSpec(2, 2)
fig = plt.figure()
ax_3d = fig.add_subplot(gs[:, 0], projection='3d')
cam_for_render_t, vert_shifted_t, joints_orig_t = vis_util.get_original(
proc_param_t, verts_t, cam_t, joints_t, img_size=img_t.shape[:2])
# Render results
ax_3d = vis_util.draw_skeleton_3d(joints3d_t, ax_3d, 'b')
ax_3d = vis_util.draw_skeleton_3d(joints3d_q, ax_3d, 'g')
ax_3d = vis_util.draw_displacement(joints3d_t, joints3d_q, ax_3d)
print("Cam param :" + str(cam_t))
print("Cam shape :" + str(cam_t.shape))
ax_3d = vis_util.draw_arrow(cam_t, [0, 0, 0], ax_3d)
## Pose 1
cam_for_render_t, vert_shifted_t, joints_orig_t = vis_util.get_original(
proc_param_t, verts_t, cam_t, joints_t, img_size=img_t.shape[:2])
# Render results
ax_img_t = fig.add_subplot(gs[0, 1])
skel_img_t = vis_util.draw_skeleton(img_t, joints_orig_t)
ax_img_t.imshow(skel_img_t)
## Pose 2
cam_for_render_q, vert_shifted_q, joints_orig_q = vis_util.get_original(
proc_param_q, verts_q, cam_q, joints_q, img_size=img_q.shape[:2])
# Render results
ax_img_q = fig.add_subplot(gs[1, 1])
skel_img_q = vis_util.draw_skeleton(img_q, joints_orig_q)
ax_img_q.imshow(skel_img_q)
plt.draw()
plt.show()
def visualize(img, proc_param, joints, verts, cam):
"""
Renders the result in original image coordinate frame.
"""
cam_for_render, vert_shifted, joints_orig = vis_util.get_original(
proc_param, verts, cam, joints, img_size=img.shape[:2])
#visualize(img, proc_param, joints[0], verts[0], cams[0])
# Render results
skel_img = vis_util.draw_skeleton(img, joints_orig)
rend_img_overlay = renderer(
#vert_shifted, cam=None, img=img, do_alpha=True)
vert_shifted, cam=cam_for_render, img=img, do_alpha=True)
rend_img = renderer(
vert_shifted, cam=cam_for_render, img_size=img.shape[:2])
rend_img_vp1 = renderer.rotated(
vert_shifted, 60, cam=cam_for_render, img_size=img.shape[:2])
rend_img_vp2 = renderer.rotated(
vert_shifted, -60, cam=cam_for_render, img_size=img.shape[:2])
import matplotlib
matplotlib.use('agg')
import matplotlib.pyplot as plt
# plt.ion()
fig = plt.figure()
plt.figure(1)
plt.clf()
plt.subplot(231)
plt.imshow(img)
plt.title('input')
plt.axis('off')
plt.subplot(232)
plt.imshow(skel_img)
plt.title('joint projection')
plt.axis('off')
plt.subplot(233)
plt.imshow(rend_img_overlay)
plt.title('3D Mesh overlay')
plt.axis('off')
plt.subplot(234)
plt.imshow(rend_img)
result = Image.fromarray(rend_img)
result.save('mesh.jpg')
plt.title('3D mesh')
plt.axis('off')
plt.subplot(235)
plt.imshow(rend_img_vp1)
plt.title('diff vp')
plt.axis('off')
plt.subplot(236)
plt.imshow(rend_img_vp2)
plt.title('diff vp')
plt.axis('off')
plt.draw()
fig.savefig('demo.jpg')
def visualize(img_path, img, proc_param, joints, verts, cam, folder_name):
"""
Renders the result in original image coordinate frame.
"""
cam_for_render, vert_shifted, joints_orig = vis_util.get_original(
proc_param, verts, cam, joints, img_size=img.shape[:2])
# Render results
skel_img = vis_util.draw_skeleton(img, joints_orig)
rend_img_overlay = renderer(
vert_shifted, cam=cam_for_render, img=img, do_alpha=True)
rend_img = renderer(
vert_shifted, cam=cam_for_render, img_size=img.shape[:2])
rend_img_vp1 = renderer.rotated(
vert_shifted, 60, cam=cam_for_render, img_size=img.shape[:2])
rend_img_vp2 = renderer.rotated(
vert_shifted, -60, cam=cam_for_render, img_size=img.shape[:2])
import matplotlib.pyplot as plt
# plt.ion()
# plt.figure(1)
# plt.clf()
# plt.subplot(231)
# plt.imshow(img)
# plt.title('input')
# plt.axis('off')
# plt.subplot(232)
# plt.imshow(skel_img)
# plt.title('joint projection')
# plt.axis('off')
# plt.subplot(233)
# plt.imshow(rend_img_overlay)
# plt.title('3D Mesh overlay')
# plt.axis('off')
# plt.subplot(234)
# plt.imshow(rend_img)
# plt.title('3D mesh')
# plt.axis('off')
# plt.subplot(235)
# plt.imshow(rend_img_vp1)
# plt.title('diff vp')
# plt.axis('off')
# plt.subplot(236)
# plt.imshow(rend_img_vp2)
# plt.title('diff vp')
# plt.axis('off')
# plt.draw()
# plt.show()
# import ipdb
# ipdb.set_trace()
#plt.imshow(rend_img_overlay)
#plt.show()
rend_img_overlay = cv2.cvtColor(rend_img_overlay, cv2.COLOR_BGR2RGB)
cv2.imwrite("/home/ankur/GUI_project/frames/FRAMES_HMR/"+ folder_name + '/' + img_path.split('/')[-1][:-4]+".png",rend_img_overlay)
def visualize(img, proc_param, joints, verts, cam, image_path):
"""
Renders the result in original image coordinate frame.
Saves result in same directory as image_path.
"""
cam_for_render, vert_shifted, joints_orig = vis_util.get_original(
proc_param, verts, cam, joints, img_size=img.shape[:2])
print(cam_for_render)
# Render results
skel_img = vis_util.draw_skeleton(img, joints_orig)
rend_img_overlay = renderer(
vert_shifted, cam=cam_for_render, img=img, do_alpha=True)
rend_img = renderer(
vert_shifted, cam=cam_for_render, img_size=img.shape[:2])
rend_img_vp1 = renderer.rotated(
vert_shifted, 60, cam=cam_for_render, img_size=img.shape[:2])
rend_img_vp2 = renderer.rotated(
vert_shifted, -60, cam=cam_for_render, img_size=img.shape[:2])
import matplotlib.pyplot as plt
# plt.ion()
plt.figure(1)
plt.clf()
plt.subplot(231)
plt.imshow(img)
plt.title('input')
plt.axis('off')
plt.subplot(232)
plt.imshow(skel_img)
plt.title('joint projection')
plt.axis('off')
plt.subplot(233)
plt.imshow(rend_img_overlay)
plt.title('3D Mesh overlay')
plt.axis('off')
plt.subplot(234)
plt.imshow(rend_img)
plt.title('3D mesh')
plt.axis('off')
plt.subplot(235)
plt.imshow(rend_img_vp1)
plt.title('diff vp')
plt.axis('off')
plt.subplot(236)
plt.imshow(rend_img_vp2)
plt.title('diff vp')
plt.axis('off')
plt.draw()
# plt.show()
plt.savefig(image_path + "hmr_result.png", format='png')
# import ipdb
# ipdb.set_trace()
plt.imsave(image_path + "hmr_overlay.png", rend_img_overlay)
def visualize(img_path, img, proc_param, joints, verts, cam, output):
"""
Renders the result in original image coordinate frame.
"""
cam_for_render, vert_shifted, joints_orig = vis_util.get_original(
proc_param, verts, cam, joints, img_size=img.shape[:2])
# Render results
skel_img = vis_util.draw_skeleton(img, joints_orig)
# TODO: add alpha to avatar
rend_img_overlay = renderer(
vert_shifted, cam=cam_for_render, img=skel_img, do_alpha=True)
# rend_img_overlay = vis_util.draw_skeleton(rend_img_overlay, joints_orig)
rend_img = renderer(
vert_shifted, cam=cam_for_render, img_size=img.shape[:2])
rend_img_vp1 = renderer.rotated(
vert_shifted, 60, cam=cam_for_render, img_size=img.shape[:2])
rend_img_vp2 = renderer.rotated(
vert_shifted, -60, cam=cam_for_render, img_size=img.shape[:2])
import matplotlib.pyplot as plt
# plt.ion()
plt.figure(1, figsize=(10, 10))
plt.clf()
plt.subplot(231)
plt.imshow(img)
plt.title('input')
plt.axis('off')
plt.subplot(232)
plt.imshow(skel_img)
plt.title('joint projection')
plt.axis('off')
plt.subplot(233)
plt.imshow(rend_img_overlay)
plt.title('3D Mesh overlay')
plt.axis('off')
plt.subplot(234)
plt.imshow(rend_img)
plt.title('3D mesh')
plt.axis('off')
plt.subplot(235)
plt.imshow(rend_img_vp1)
plt.title('diff vp')
plt.axis('off')
plt.subplot(236)
plt.imshow(rend_img_vp2)
plt.title('diff vp')
plt.axis('off')
plt.draw()
print('saving to %s' % output)
plt.savefig(os.path.join(output, os.path.splitext(os.path.basename(img_path))[0] + ".png"))
io.imsave(os.path.join(output, os.path.splitext(os.path.basename(img_path))[0] + "_big.png"), rend_img_overlay) # rend_img[:,:,:3])#
def visualize(img, proc_param, joints, verts, cam):
"""
Renders the result in original image coordinate frame.
"""
cam_for_render, vert_shifted, joints_orig = vis_util.get_original(
proc_param, verts, cam, joints, img_size=img.shape[:2])
# Render results
rend_img = renderer(vert_shifted,
cam=cam_for_render,
img_size=img.shape[:2])
return rend_img
def visualize(img, proc_param, joints, verts, cam, filename):
"""
Renders the result in original image coordinate frame.
"""
cam_for_render, vert_shifted, joints_orig = vis_util.get_original(
proc_param, verts, cam, joints, img_size=img.shape[:2])
# Render results
skel_img = vis_util.draw_skeleton(img, joints_orig)
rend_img_overlay = renderer(vert_shifted,
cam=cam_for_render,
img=img,
do_alpha=True)
rend_img = renderer(vert_shifted,
cam=cam_for_render,
img_size=img.shape[:2])
rend_img_vp1 = renderer.rotated(vert_shifted,
60,
cam=cam_for_render,
img_size=img.shape[:2])
rend_img_vp2 = renderer.rotated(vert_shifted,
-60,
cam=cam_for_render,
img_size=img.shape[:2])
import matplotlib.pyplot as plt
plt.clf()
plt.imshow(img)
plt.title('input')
plt.axis('off')
plt.savefig("exp/figure/" + filename + "-a.png",
format='png',
transparent=True,
dpi=800)
plt.clf()
plt.imshow(skel_img)
plt.title('joint projection')
plt.axis('off')
plt.savefig("exp/figure/" + filename + "-b.png",
format='png',
transparent=True,
dpi=800)
plt.clf()
plt.imshow(rend_img_overlay)
plt.title('3D Mesh overlay')
plt.axis('off')
plt.savefig("exp/figure/" + filename + "-c.png",
format='png',
transparent=True,
dpi=800)
def visualize(proc_param, renderer, joints, verts, cam, img_size=(230, 230)):
"""
Renders the result in original image coordinate frame.
"""
cam_for_render, vert_shifted, joints_orig = vis_util.get_original(
proc_param, verts, cam, joints, img_size=img_size)
# Render results
# rend_img = renderer(verts, cam=cam, img_size=img_size)
rend_img = renderer(vert_shifted, cam=cam_for_render, img_size=img_size)
# rend_img = renderer.rotated(vert_shifted, 90, cam=cam_for_render, img_size=img_size)
fig = plt.figure(1)
plt.imshow(rend_img)
return fig
def visualize(self, img, proc_param, joints, verts, cam):
cam_for_render, vert_shifted, joints_orig = vis_util.get_original(
proc_param, verts, cam, joints, img_size=img.shape[:2])
rend_img_overlay = self.renderer(vert_shifted,
cam=cam_for_render,
img=img,
do_alpha=True)
image3d = Image.fromarray(rend_img_overlay)
image3d = image3d.resize((224, 224), Image.ANTIALIAS)
render = ImageTk.PhotoImage(image3d)
otherimg = Label(self, image=render)
otherimg.image = render
otherimg.place(x=1, y=1)
def get_silhouette(sil, joints, verts, cam):
_, proc_param, _ = preprocess_image(sil, ignore_read=True)
cam_for_render, vert_shifted, joints_orig = vis_util.get_original(
proc_param, verts, cam, joints)
renderer = vis_util.SMPLRenderer()
# This returns the image with black image silhouette and white background in 255 format
rend_img = renderer(vert_shifted,
cam=cam_for_render,
img_size=proc_param['img_size'][:2],
light_color=np.array([0, 0, 0]))
return rend_img
def main(folder_path, json_path=None):
sess = tf.Session()
model = RunModel(config, sess=sess)
images = sorted([f for f in listdir(folder_path) if f.endswith('image.png')])
annotated_joints = sorted([f for f in listdir(folder_path) if f.endswith('.npy')])
abs_errors = []
num_joints = 0
num_accepted = 0
images_joints_dict = dict(zip(images, annotated_joints))
print('Predicting on all png images in folder.')
for image in images:
print('Image:', image)
img_path = join(folder_path, image)
input_img, proc_param, img = preprocess_image(img_path, json_path)
# Add batch dimension: 1 x D x D x 3
input_img = np.expand_dims(input_img, 0)
joints, verts, cams, joints3d, theta = model.predict(
input_img, get_theta=True)
cam_for_render, vert_shifted, joints_orig = vis_util.get_original(
proc_param, verts[0], cams[0], joints[0], img_size=img.shape[:2])
annotation_array = np.load(join(folder_path, images_joints_dict[image]))
annotation_array = zip(annotation_array[0], annotation_array[1])
annotation_array = (np.rint(annotation_array)).astype(int)
joints_orig = (np.rint(joints_orig)).astype(int)
joints_orig = joints_orig[:14]
for i in range(len(annotation_array)):
annotation = (annotation_array[i][0], annotation_array[i][1])
prediction = (joints_orig[i][0], joints_orig[i][1])
# cv2.circle(img, annotation, 2,
# (0, 255, 0), thickness=5)
# cv2.circle(img, prediction, 2,
# (0, 0, 255), thickness=5)
# cv2.imshow('win', img)
# cv2.waitKey(0)
error = np.linalg.norm(np.subtract(annotation, prediction))
# print(error)
abs_errors.append(error)
if error < 30:
num_accepted += 1
num_joints += 1
print("MAE", np.mean(abs_errors), 'Fraction Accepted', float(num_accepted)/num_joints)