def run_worker(self):
ds_1, mesh_1, candidates = self.inputs
dists = np.asarray([parent.load() for parent in self.parents()])
min_idx = np.argmin(dists)
ds_2, mesh_2 = candidates[min_idx * 2], candidates[min_idx * 2 + 1]
pc1 = SamplerTask(self.fs, (ds_1, mesh_1)).load()
pc2 = SamplerTask(self.fs, (ds_2, mesh_2)).load()
T, _, _ = icp.icp(pc1, pc2)
return (min_idx, mesh_2, T)
def calculate_metrics(models,
batches,
pcl_gt_scaled,
pred_scaled,
indices=None):
if FLAGS.visualize:
iters = range(batches)
else:
iters = tqdm(range(batches))
epoch_chamfer = 0.
epoch_forward = 0.
epoch_backward = 0.
epoch_emd = 0.
ph_gt = tf.placeholder(tf.float32, (BATCH_SIZE, NUM_EVAL_POINTS, 3),
name='ph_gt')
ph_pr = tf.placeholder(tf.float32, (BATCH_SIZE, NUM_EVAL_POINTS, 3),
name='ph_pr')
dists_forward, dists_backward, chamfer_distance = get_chamfer_metrics(
ph_gt, ph_pr)
emd = get_emd_metrics(ph_gt, ph_pr, BATCH_SIZE, NUM_EVAL_POINTS)
for cnt in iters:
start = time.time()
if FLAGS.dataset == 'shapenet':
batch_ip, batch_gt = fetch_batch_shapenet(models, indices, cnt,
BATCH_SIZE)
elif FLAGS.dataset == 'pix3d':
batch_ip, batch_gt = fetch_batch_pix3d(models, cnt, BATCH_SIZE)
_gt_scaled, _pr_scaled = sess.run([pcl_gt_scaled, pred_scaled],
feed_dict={
pcl_gt: batch_gt,
img_inp: batch_ip
})
_pr_scaled_icp = []
for i in xrange(BATCH_SIZE):
rand_indices = np.random.permutation(NUM_POINTS)[:NUM_EVAL_POINTS]
T, _, _ = icp(_gt_scaled[i],
_pr_scaled[i][rand_indices],
tolerance=1e-10,
max_iterations=1000)
_pr_scaled_icp.append(
np.matmul(_pr_scaled[i][rand_indices], T[:3, :3]) - T[:3, 3])
_pr_scaled_icp = np.array(_pr_scaled_icp).astype('float32')
C, F, B, E = sess.run(
[chamfer_distance, dists_forward, dists_backward, emd],
feed_dict={
ph_gt: _gt_scaled,
ph_pr: _pr_scaled_icp
})
epoch_chamfer += C.mean() / batches
epoch_forward += F.mean() / batches
epoch_backward += B.mean() / batches
epoch_emd += E.mean() / batches
if FLAGS.visualize:
for i in xrange(BATCH_SIZE):
print '-' * 50
print C[i], F[i], B[i], E[i]
print '-' * 50
cv2.imshow('', batch_ip[i])
print 'Displaying Gt scaled 1k'
show3d_balls.showpoints(_gt_scaled[i], ballradius=3)
print 'Displaying Pr scaled icp 1k'
show3d_balls.showpoints(_pr_scaled_icp[i], ballradius=3)
if cnt % 10 == 0:
print '%d / %d' % (cnt, batches)
if not FLAGS.visualize:
log_values(csv_path, epoch_chamfer, epoch_forward, epoch_backward,
epoch_emd)
return
def run_worker(self):
pc1, pc2 = [parent.load() for parent in self.parents()]
T, _, _ = icp.icp(pc1, pc2)
return T
def register_imgs(img1_rgb,
img2_rgb,
img1_depth,
img2_depth,
scale=1.,
filter_pts_frac=1.,
partial_set_frac=1.,
img1_pts=None,
img2_pts=None,
plot=False):
"""
Perform global image registration given the RGB and depth of two images by
1. Performing global registration by extracting keypoints from RGB images
2. Performing local registration by ICP
:param img1_rgb: (h, w, 3) image
:param img2_rgb: (h, w, 3) image
:param img1_depth: (h, w) depth image
:param img2_depth: (h, w) depth image
:param scale: scaling factor for loading point clouds (see in-depth explanation in depth_to_voxel() function)
:param filter_pts_frac: fraction of all the points to use when performing ICP. Must be in range (0, 1], though
choosing a good value would depend on the density of the depth we are working with.
:param partial_set_frac: fraction of expected overlap between the two images when performing ICP. Must be in
range (0, 1]. In the case of video data, this value should usually be close to 1,
though maybe not exactly 1, since we can expect a high overlap between frames of a
video. Alternatively, if we are taking one frame from every couple of frames, this
value should be slightly lower.
:param img1_pts: (h x w, 3) points, optional. If this is given, scale is not needed for image 1.
:param img2_pts: (h x w, 3) points, optional. If this is given, scale is not needed for image 1.
:param plot: True to draw matches, False otherwise
:return: image 1 point cloud,
image 2 point cloud,
4x4 transformation matrix that maps image 1 to image 2
"""
# convert depth to point cloud
if img1_pts is None:
img1_pts = depth_to_voxel(img1_depth, scale=scale)
if img2_pts is None:
img2_pts = depth_to_voxel(img2_depth, scale=scale)
# find RGB matches
kp1, kp2, matches = generate_keypoints_and_match(img1_rgb, img2_rgb)
matches = np.array([(m.queryIdx, m.trainIdx) for m in matches])
_, matches = ransac_loop(img1_rgb, img2_rgb, kp1, kp2, matches)
# draw matches
if plot:
np_kp1 = np.array([(kp.pt[1], kp.pt[0]) for kp in kp1])
np_kp2 = np.array([(kp.pt[1], kp.pt[0]) for kp in kp2])
draw_matches(img1_rgb,
img2_rgb,
np_kp1,
np_kp2,
matches,
title="Matches after RANSAC")
# get 3D coordinates of matches
kp1, kp2 = get_key_points_from_matches(kp1, kp2, matches)
img1_kp = [(p[0], p[1], img1_depth[p[1], p[0]]) for p in kp1]
img2_kp = [(p[0], p[1], img2_depth[p[1], p[0]]) for p in kp2]
# remove 0 entries
zipped = zip(img1_kp, img2_kp)
zipped = [z for z in zipped if z[0][2] != 0 and z[1][2] != 0]
img1_kp, img2_kp = zip(*zipped)
img1_kp = np.array(img1_kp)
img2_kp = np.array(img2_kp)
# find transformation
h = homo_rigid_transform_3d(img1_kp, img2_kp)
img2_new = get_transformed_points(img1_pts, h)
# filter points (since we have a dense point cloud)
pts_idx1 = np.random.choice(img2_new.shape[0],
int(img2_new.shape[0] * filter_pts_frac),
replace=False)
pts_idx2 = np.random.choice(img2_pts.shape[0],
int(img2_pts.shape[0] * filter_pts_frac),
replace=False)
# ICP for fine registration
t, _, iter_to_converge = icp(img2_new[pts_idx1],
img2_pts[pts_idx2],
tolerance=0.001,
max_iterations=100,
partial_set_size=partial_set_frac)
# return point clouds and transformation
return img1_pts, img2_pts, np.dot(h, t)
cam_params = np.loadtxt(view_path)
idx = int(float(file_list_sub[2][:-4]))
cam_params = cam_params[idx]
cam_mat, cam_pos = camera_info(cam_params)
pcl_pred = np.dot(pcl_pred, cam_mat) + cam_pos
x = np.expand_dims(pcl_pred[:, 0], axis=1)
y = np.expand_dims(pcl_pred[:, 1], axis=1)
z = np.expand_dims(pcl_pred[:, 2], axis=1)
pcl_pred = np.concatenate((-y, -z, x), axis=1)
pcl_pred = pcl_pred / 0.57
# load gt
gt_path = data_dir + 'ShapeNet_pointclouds/' + class_id + '/' + file_list_sub[
1] + '/pointcloud_1024.npy'
pcl_gt = np.load(gt_path)
# Perform Scaling
pcl_gt_scaled, pcl_pred_scaled = sess.run([xyz3_scaleds, xyz4_scaleds],
feed_dict={
xyz1: pcl_gt,
xyz2: pcl_pred
})
T, _, _ = icp(pcl_gt_scaled,
pcl_pred_scaled,
tolerance=1e-10,
max_iterations=1000)
pcl_pred_icp = np.matmul(pcl_pred_scaled, T[:3, :3]) - T[:3, 3]
np.savetxt(eval_path + file_list[19:-4] + '_pred.npy', pcl_pred_icp)
np.savetxt(eval_path + file_list[19:-4] + '_gt.npy', pcl_gt_scaled)
for cnt in iters:
start = time.time()
batch_ip, batch_gt = fetch_batch_shapenet(models, indices, cnt, BATCH_SIZE)
_gt_scaled, _pr_scaled = sess.run(
[pcl_gt_scaled, pred_scaled],
feed_dict={pcl_in:batch_ip, pcl_gt:batch_gt, img_inp:batch_img}
)
_pr_scaled_icp = []
for i in xrange(BATCH_SIZE):
rand_indices = np.random.permutation(NUM_POINTS)[:NUM_EVAL_POINTS]
T, _, _ = icp(_gt_scaled[i], _pr_scaled[i][rand_indices], tolerance=1e-10, max_iterations=1000)
_pr_scaled_icp.append(np.matmul(_pr_scaled[i][rand_indices], T[:3,:3]) - T[:3, 3])
_pr_scaled_icp = np.array(_pr_scaled_icp).astype('float32')
C,F,B,E = sess.run(
[chamfer_distance, dists_forward, dists_backward, emd],
feed_dict={ph_gt:_gt_scaled, ph_pr:_pr_scaled_icp}
)
epoch_chamfer += C.mean() / batches
epoch_forward += F.mean() / batches
epoch_backward += B.mean() / batches
epoch_emd += E.mean() / batches
if FLAGS.visualize: