def pcd2img(name, load_path, save_path, k, factor_path=None):
"""Convert point clouds to depth images.
"""
mask = IO.get(os.path.join(load_path, "%s/instance_segment.png" % name))
mask_bg = np.array(mask[:, :, 2] == 0, dtype=np.float32)
depth_in = IO.get(os.path.join(load_path, "%s/depth.exr" % name))
depth_out = depth_in * mask_bg
mask_vals = np.unique(mask[:, :, 2])[1:]
for i in range(len(mask_vals)):
pcd_i = IO.get(
os.path.join(load_path, "%s/depth2pcd_pred_%d.pcd" % (name, i)))
if factor_path is not None:
with open(
os.path.join(factor_path,
"%s/scale_factor_%d.json" % (name, i))) as f:
factors = json.loads(f.read())
if factors['normalize']:
pcd_i *= float(factors['normalize_factor'])
if factors['centering']:
pcd_i += np.array(factors['center_position'])
depth_i = project_to_image(k, pcd_i)
mask_i = np.array(mask[:, :, 2] == mask_vals[i], dtype=np.float32)
depth_out += depth_i * mask_i
# write predicted depth image in exr
depth_out = np.expand_dims(depth_out, -1).repeat(3, axis=2)
pyexr.write(os.path.join(save_path, "%s/depth_pred.exr" % name), depth_out)
def rms(path, gt_name, pred_name):
""" Compute the average root-mean-squared error of predicted depth images.
"""
errors = []
for subdir, dirs, files in os.walk(path):
for dirname in sorted(dirs):
depth_gt = IO.get(os.path.join(path, '%s/%s' % (dirname, gt_name)))
depth_pred = IO.get(os.path.join(path, '%s/%s' % (dirname, pred_name)))
diff = np.where(depth_pred > 0, depth_pred - depth_gt, 0)
errors.append(np.sqrt(np.mean(diff**2)))
return np.array(errors).mean(), np.array(errors).sum()
def img2pcd(name, load_path, save_path, inv_k, normalize=False, plot=False):
"""Convert a pair of ClearGrasp images with opaque and transparent objects into point clouds.
"""
mask = IO.get(os.path.join(load_path, "%s-mask.png" % name))
# Separate multiple objects into multiple point clouds
mask_copy = np.array(mask * 255, dtype=np.uint8)
if plot:
cv2.imshow("%s mask" % name, mask_copy)
cv2.waitKey(0)
contours, hierarchy = cv2.findContours(mask_copy, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
obj_idx = 0
for i in range(len(contours)):
if cv2.contourArea(contours[i]) > 100:
mask_i = np.zeros_like(mask)
cv2.drawContours(mask_i, contours, contourIdx=i, color=255, thickness=-1)
if plot:
cv2.imshow("%s idx:%d" % (name, i), mask_i)
cv2.waitKey(0)
mask_pcd = deproject(mask_i / 255, inv_k).sum(axis=1) > 0
opaque_depth = IO.get(os.path.join(load_path, "%s-opaque-depth-img.exr" % name))
opaque_pcd = deproject(opaque_depth, inv_k)[mask_pcd]
IO.put(os.path.join(save_path, "opaque/%s-%d.pcd" % (name, obj_idx)), opaque_pcd)
transp_depth = IO.get(os.path.join(load_path, "%s-transparent-depth-img.exr" % name))
transp_pcd = deproject(transp_depth, inv_k)[mask_pcd]
IO.put(os.path.join(save_path, "transparent/%s-%d.pcd" % (name, obj_idx)), transp_pcd)
if normalize: # normalize pcd to be within [-1, 1] for gridding
max_val = np.nanmax((np.max(np.abs(opaque_pcd)), np.max(np.abs(transp_pcd))))
if max_val > 1:
print(max_val)
opaque_pcd /= max_val * 1.01
transp_pcd /= max_val * 1.01
with open(os.path.join(save_path, "norm_factors/%s-%d.json" % (name, obj_idx)), 'w') as outfile:
json.dump(max_val * 1.01, outfile)
obj_idx += 1
# print(mask_pcd.sum())
# print(opaque_pcd.shape)
# print(transp_pcd.shape)
else:
if plot:
mask_i = np.zeros_like(mask)
cv2.drawContours(mask_i, contours, contourIdx=i, color=255, thickness=5)
cv2.imshow("%s idx:%d" % (name, i), mask_i)
cv2.waitKey(0)
def img2pcd(name,
load_path,
save_path,
inv_k,
centering=False,
normalize=False,
skip=False,
plot=False):
"""Convert a pair of ClearGrasp images with opaque and transparent objects into point clouds.
"""
mask = IO.get(os.path.join(load_path, "%s/instance_segment.png" % name))
# Separate multiple objects into multiple point clouds
mask_vals = np.unique(mask[:, :, 2])[
1:] # each object is indicated by a distinct value in RED channel
maxdis = []
skip_count = 0
for i in range(len(mask_vals)):
mask_i = np.array(mask[:, :, 2] == mask_vals[i], dtype=np.float32)
if plot:
cv2.imshow("%s idx:%d" % (name, i), mask_i)
cv2.waitKey(0)
mask_pcd = deproject(mask_i, inv_k).sum(axis=1) > 0
# opaque_depth = IO.get(os.path.join(load_path, "%s/detph_GroundTruth.exr" % name))
# opaque_pcd = deproject(opaque_depth, inv_k)[mask_pcd]
opaque_pcd = IO.get(
os.path.join(load_path, "%s/Ground_Truth_%d.pcd" % (name, i)))
transp_depth = IO.get(os.path.join(load_path, "%s/depth.exr" % name))
transp_pcd = deproject(transp_depth, inv_k)[mask_pcd]
center = transp_pcd.mean(axis=0)
if centering:
opaque_pcd -= center
transp_pcd -= center
maxdis.append(
np.max((np.max(np.abs(opaque_pcd)), np.max(np.abs(transp_pcd)))))
if normalize and not skip:
opaque_pcd /= maxdis[-1] * 1.01
transp_pcd /= maxdis[-1] * 1.01
if maxdis[-1] >= 1 and skip:
skip_count += 1
continue
if not os.path.exists(os.path.join(save_path, name)):
os.mkdir(os.path.join(save_path, name))
if maxdis[-1] == 0:
print((name, i))
# save centering and scaling factors
factors = {
'centering': centering,
'center_position': center.tolist(),
'normalize': normalize,
'normalize_factor': maxdis[-1] * 1.01,
}
with open(
os.path.join(save_path, "%s/scale_factor_%d.json" % (name, i)),
'w') as outfile:
json.dump(factors, outfile)
# IO.put(os.path.join(save_path, "%s/depth2pcd_GT_%d.pcd" % (name, i)), opaque_pcd)
IO.put(
os.path.join(save_path,
"%s/Ground_Truth_recenter_%d.pcd" % (name, i)),
opaque_pcd)
IO.put(os.path.join(save_path, "%s/depth2pcd_%d.pcd" % (name, i)),
transp_pcd)
# print(mask_pcd.sum())
# print(opaque_pcd.shape)
# print(transp_pcd.shape)
return np.max(maxdis), skip_count
opaque = o3d.io.read_point_cloud("./%s/%s/depth2pcd_GT_%d.pcd" %
(dset, name, obj_idx))
print(np.array(opaque.points).shape)
o3d.visualization.draw_geometries([opaque])
opaque = o3d.io.read_point_cloud("./%s/%s/Ground_Truth_recenter_%d.pcd" %
(dset, name, obj_idx))
print(np.array(opaque.points).shape)
o3d.visualization.draw_geometries([opaque])
pred_i = o3d.io.read_point_cloud("./%s/%s/depth2pcd_pred_%d.pcd" %
(dset, name, obj_idx))
print(np.array(pred_i.points).shape)
o3d.visualization.draw_geometries([pred_i])
depth_raw = IO.get("./%s/%s/depth.exr" % (dset, name))
plt.imshow(depth_raw)
plt.show()
pt_raw = deproject(depth_raw, INV_K)
pcd = o3d.geometry.PointCloud()
pcd.points = o3d.utility.Vector3dVector(pt_raw)
print(np.array(pcd.points).shape)
o3d.visualization.draw_geometries([pcd])
depth_gt = IO.get("./%s/%s/detph_GroundTruth.exr" % (dset, name))
plt.imshow(depth_gt)
plt.show()
pt_gt = deproject(depth_gt, INV_K)
pcd = o3d.geometry.PointCloud()
pcd.points = o3d.utility.Vector3dVector(pt_gt)
print(np.array(pcd.points).shape)