def project2D2(r, z, a, e, R_c, scale):
"""Project 3D points into 2D and return them in the same scale as the given
2D predicted points."""
mod = uc.build_model(a, e, z)
p = uc.project_model(mod, R_c, r)
p *= scale[:,np.newaxis,np.newaxis]
return p
def project2D2(r, z, a, e, R_c, scale):
"""Project 3D points into 2D and return them in the same scale as the given
2D predicted points."""
mod = uc.build_model(a, e, z)
p = uc.project_model(mod, R_c, r)
p *= scale[:, np.newaxis, np.newaxis]
return p
def executeOnFrame(NN, net, data, output_dir, proj=False, astr=False,
show=False, stage_3d_err=False, hm_joint=False):
num_channels = ut.getNumChannelsLayer(net,'data')
joints = np.array(data['joint_self'])
# Get Image
img_orig = ut.cv2.imread(data['img_paths'])
box_points = ut.getBoundingBox(joints, ut.getCenterJoint(joints),
75, img_orig.shape[1], img_orig.shape[0])
img = ut.cropImage(img_orig, box_points)
save_name = output_dir + 'image_RGB.png'
if astr:
save_name = output_dir + 'image_RGB' + astr + '.png'
img_saved = ut.convertImgCv2(img)
ut.plt.imsave(save_name, img_saved)
# Extract 2D predictions
(imgch, info, resizedImage) = preprocessImage(NN, data, 1, num_channels)
ut.netForward(net, imgch)
img = ut.convertImgCv2(img)
layer_name = 'Mconv5_stage6_new'
out = ut.getOutputLayer(net, layer_name)
(pred, heatMaps, err) = postprocessHeatmaps(NN, out, data, info)
img_2d_skel = ut.plotImageJoints(img_orig, pred, h=(not show))
img_2d_skel = ut.cropImage(img_2d_skel, box_points)
save_name = output_dir + 'image_2d.png'
if astr:
save_name = output_dir + 'image_2d' + astr + '.png'
ut.plt.imsave(save_name, img_2d_skel)
# Plot data
heatMap = ut.cropImage(heatMaps[:,:,-1], box_points)
save_name = output_dir + 'image_hm.png'
if astr:
save_name = output_dir + 'image_hm' + astr + '.png'
ut.plt.imsave(save_name, heatMap)
Lambda = 0.05
(default_r, e, z, weights) = load_parameters()
(w, s, mean) = normalise_data(pred.flatten())
w = w[np.newaxis,:]
camera = int(data['camera']) - 1
(a, r) = uc.estimate_a_and_r(w, e, z, default_r[camera], Lambda*weights)
for j in xrange(10):
r = uc.reestimate_r(w, z, a, e, default_r[camera], r)
(a, res) = uc.reestimate_a(w, e, z, r, default_r[camera], Lambda*weights)
mod = uc.build_model(a, e, z).squeeze()
save_name = output_dir + 'image_3d.pdf'
if astr:
save_name = output_dir + 'image_3d' + astr + '.pdf'
ut.plot3DJoints(-mod, save_pdf=save_name)
#ut.plot3DJoints(-mod, pbaspect=[1,0.88,1])
if stage_3d_err:
resizedImage = ut.convertImgCv2(resizedImage)/255.0
layers = ['conv7_stage1_new', 'Mconv5_stage2_new',
'Mconv5_stage3_new', 'Mconv5_stage4_new',
'Mconv5_stage5_new', 'Mconv5_stage6_new']
gt3d = ut.filterJoints(data['joint_self_3d'])
for l in range(len(layers)):
layer_name = layers[l]
out = ut.getOutputLayer(net, layer_name)
(pred, _, _) = postprocessHeatmaps(NN, out, data, info)
if hm_joint:
# save heat-map
heatMap = ut.cv2.resize(out[hm_joint], (NN['inputSize'],NN['inputSize']),
interpolation = ut.cv2.INTER_LANCZOS4)
save_name = 'hm_joint_%d_stage_%d.png' % (hm_joint, l)
ut.plt.imsave(output_dir+save_name, heatMap)
# generate heat-maps
channels = out.transpose(1,2,0)
hm = ut.cv2.resize(channels, (resizedImage.shape[0],resizedImage.shape[0]),
interpolation = ut.cv2.INTER_LANCZOS4)
hm = hm[:,:,hm_joint]
ut.plt.imshow(resizedImage)
ut.plt.hold(True)
ut.plt.imshow(hm,alpha=0.6) #color='Blues')
save_name = 'hm_joint_%d_stage_%d.pdf' % (hm_joint, l)
ut.plt.axis('off')
ut.plt.savefig(output_dir+save_name)
ut.plt.close()
# generate skeletons
save_name = 'img_skel_stage_%d.png' % (l)
img_2d_skel = ut.plotImageJoints(img_orig, pred, h=False)
new_box_points = box_points.copy()
new_box_points[0] += 120
new_box_points[2] -= 240
new_box_points[1] += 70
new_box_points[3] -= 70
img_2d_skel = ut.cropImage(img_2d_skel, new_box_points)
ut.plt.imsave(output_dir+save_name, img_2d_skel)
(w, s, mean), (w3d, s3d) = normalise_data(pred.flatten(), w3d=gt3d)
w = w[np.newaxis,:]
(a, r) = uc.estimate_a_and_r(w, e, z, default_r[camera], Lambda*weights)
for j in xrange(10):
r = uc.reestimate_r(w, z, a, e, default_r[camera], r)
(a, res) = uc.reestimate_a(w, e, z, r, default_r[camera], Lambda*weights)
m = uc.build_and_rot_model(a,e,z,r).squeeze()
err = cost3d(m, w3d, s3d)
print '3D error stage %r: %.2f' % (l, err/17.0)
save_name = '%sstage_%d_3d.pdf' % (output_dir, l)
title = 'Stage %r\nErr: %.2f mm' % (l+1, err/17.0)
ut.plot3DJoints(-m, save_pdf=save_name, title=title)
if proj:
points = project2D(r, z, a, e, default_r[camera], s).squeeze()
points += mean[:,np.newaxis]
ut.plt.figure()
img_2d_skel = ut.plotImageJoints(img_orig, points.T, h=False)
img_2d_skel = ut.cropImage(img_2d_skel, box_points)
save_name = output_dir + 'image_2d_proj.png'
if astr:
save_name = output_dir + 'image_2d_proj' + astr + '.png'
ut.plt.imsave(save_name, img_2d_skel)
def executeOnFrame(NN,
net,
data,
output_dir,
proj=False,
astr=False,
show=False,
stage_3d_err=False,
hm_joint=False):
num_channels = ut.getNumChannelsLayer(net, 'data')
joints = np.array(data['joint_self'])
# Get Image
img_orig = ut.cv2.imread(data['img_paths'])
box_points = ut.getBoundingBox(joints, ut.getCenterJoint(joints), 75,
img_orig.shape[1], img_orig.shape[0])
img = ut.cropImage(img_orig, box_points)
save_name = output_dir + 'image_RGB.png'
if astr:
save_name = output_dir + 'image_RGB' + astr + '.png'
img_saved = ut.convertImgCv2(img)
ut.plt.imsave(save_name, img_saved)
# Extract 2D predictions
(imgch, info, resizedImage) = preprocessImage(NN, data, 1, num_channels)
ut.netForward(net, imgch)
img = ut.convertImgCv2(img)
layer_name = 'Mconv5_stage6_new'
out = ut.getOutputLayer(net, layer_name)
(pred, heatMaps, err) = postprocessHeatmaps(NN, out, data, info)
img_2d_skel = ut.plotImageJoints(img_orig, pred, h=(not show))
img_2d_skel = ut.cropImage(img_2d_skel, box_points)
save_name = output_dir + 'image_2d.png'
if astr:
save_name = output_dir + 'image_2d' + astr + '.png'
ut.plt.imsave(save_name, img_2d_skel)
# Plot data
heatMap = ut.cropImage(heatMaps[:, :, -1], box_points)
save_name = output_dir + 'image_hm.png'
if astr:
save_name = output_dir + 'image_hm' + astr + '.png'
ut.plt.imsave(save_name, heatMap)
Lambda = 0.05
(default_r, e, z, weights) = load_parameters()
(w, s, mean) = normalise_data(pred.flatten())
w = w[np.newaxis, :]
camera = int(data['camera']) - 1
(a, r) = uc.estimate_a_and_r(w, e, z, default_r[camera], Lambda * weights)
for j in xrange(10):
r = uc.reestimate_r(w, z, a, e, default_r[camera], r)
(a, res) = uc.reestimate_a(w, e, z, r, default_r[camera],
Lambda * weights)
mod = uc.build_model(a, e, z).squeeze()
save_name = output_dir + 'image_3d.pdf'
if astr:
save_name = output_dir + 'image_3d' + astr + '.pdf'
ut.plot3DJoints(-mod, save_pdf=save_name)
#ut.plot3DJoints(-mod, pbaspect=[1,0.88,1])
if stage_3d_err:
resizedImage = ut.convertImgCv2(resizedImage) / 255.0
layers = [
'conv7_stage1_new', 'Mconv5_stage2_new', 'Mconv5_stage3_new',
'Mconv5_stage4_new', 'Mconv5_stage5_new', 'Mconv5_stage6_new'
]
gt3d = ut.filterJoints(data['joint_self_3d'])
for l in range(len(layers)):
layer_name = layers[l]
out = ut.getOutputLayer(net, layer_name)
(pred, _, _) = postprocessHeatmaps(NN, out, data, info)
if hm_joint:
# save heat-map
heatMap = ut.cv2.resize(out[hm_joint],
(NN['inputSize'], NN['inputSize']),
interpolation=ut.cv2.INTER_LANCZOS4)
save_name = 'hm_joint_%d_stage_%d.png' % (hm_joint, l)
ut.plt.imsave(output_dir + save_name, heatMap)
# generate heat-maps
channels = out.transpose(1, 2, 0)
hm = ut.cv2.resize(
channels, (resizedImage.shape[0], resizedImage.shape[0]),
interpolation=ut.cv2.INTER_LANCZOS4)
hm = hm[:, :, hm_joint]
ut.plt.imshow(resizedImage)
ut.plt.hold(True)
ut.plt.imshow(hm, alpha=0.6) #color='Blues')
save_name = 'hm_joint_%d_stage_%d.pdf' % (hm_joint, l)
ut.plt.axis('off')
ut.plt.savefig(output_dir + save_name)
ut.plt.close()
# generate skeletons
save_name = 'img_skel_stage_%d.png' % (l)
img_2d_skel = ut.plotImageJoints(img_orig, pred, h=False)
new_box_points = box_points.copy()
new_box_points[0] += 120
new_box_points[2] -= 240
new_box_points[1] += 70
new_box_points[3] -= 70
img_2d_skel = ut.cropImage(img_2d_skel, new_box_points)
ut.plt.imsave(output_dir + save_name, img_2d_skel)
(w, s, mean), (w3d, s3d) = normalise_data(pred.flatten(), w3d=gt3d)
w = w[np.newaxis, :]
(a, r) = uc.estimate_a_and_r(w, e, z, default_r[camera],
Lambda * weights)
for j in xrange(10):
r = uc.reestimate_r(w, z, a, e, default_r[camera], r)
(a, res) = uc.reestimate_a(w, e, z, r, default_r[camera],
Lambda * weights)
m = uc.build_and_rot_model(a, e, z, r).squeeze()
err = cost3d(m, w3d, s3d)
print '3D error stage %r: %.2f' % (l, err / 17.0)
save_name = '%sstage_%d_3d.pdf' % (output_dir, l)
title = 'Stage %r\nErr: %.2f mm' % (l + 1, err / 17.0)
ut.plot3DJoints(-m, save_pdf=save_name, title=title)
if proj:
points = project2D(r, z, a, e, default_r[camera], s).squeeze()
points += mean[:, np.newaxis]
ut.plt.figure()
img_2d_skel = ut.plotImageJoints(img_orig, points.T, h=False)
img_2d_skel = ut.cropImage(img_2d_skel, box_points)
save_name = output_dir + 'image_2d_proj.png'
if astr:
save_name = output_dir + 'image_2d_proj' + astr + '.png'
ut.plt.imsave(save_name, img_2d_skel)