def img2pcd(name, load_path, save_path, inv_k, normalize=False, plot=False): """Convert a pair of ClearGrasp images with opaque and transparent objects into point clouds. """ mask = IO.get(os.path.join(load_path, "%s-mask.png" % name)) # Separate multiple objects into multiple point clouds mask_copy = np.array(mask * 255, dtype=np.uint8) if plot: cv2.imshow("%s mask" % name, mask_copy) cv2.waitKey(0) contours, hierarchy = cv2.findContours(mask_copy, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) obj_idx = 0 for i in range(len(contours)): if cv2.contourArea(contours[i]) > 100: mask_i = np.zeros_like(mask) cv2.drawContours(mask_i, contours, contourIdx=i, color=255, thickness=-1) if plot: cv2.imshow("%s idx:%d" % (name, i), mask_i) cv2.waitKey(0) mask_pcd = deproject(mask_i / 255, inv_k).sum(axis=1) > 0 opaque_depth = IO.get(os.path.join(load_path, "%s-opaque-depth-img.exr" % name)) opaque_pcd = deproject(opaque_depth, inv_k)[mask_pcd] IO.put(os.path.join(save_path, "opaque/%s-%d.pcd" % (name, obj_idx)), opaque_pcd) transp_depth = IO.get(os.path.join(load_path, "%s-transparent-depth-img.exr" % name)) transp_pcd = deproject(transp_depth, inv_k)[mask_pcd] IO.put(os.path.join(save_path, "transparent/%s-%d.pcd" % (name, obj_idx)), transp_pcd) if normalize: # normalize pcd to be within [-1, 1] for gridding max_val = np.nanmax((np.max(np.abs(opaque_pcd)), np.max(np.abs(transp_pcd)))) if max_val > 1: print(max_val) opaque_pcd /= max_val * 1.01 transp_pcd /= max_val * 1.01 with open(os.path.join(save_path, "norm_factors/%s-%d.json" % (name, obj_idx)), 'w') as outfile: json.dump(max_val * 1.01, outfile) obj_idx += 1 # print(mask_pcd.sum()) # print(opaque_pcd.shape) # print(transp_pcd.shape) else: if plot: mask_i = np.zeros_like(mask) cv2.drawContours(mask_i, contours, contourIdx=i, color=255, thickness=5) cv2.imshow("%s idx:%d" % (name, i), mask_i) cv2.waitKey(0)
def img2pcd(name, load_path, save_path, inv_k, centering=False, normalize=False, skip=False, plot=False): """Convert a pair of ClearGrasp images with opaque and transparent objects into point clouds. """ mask = IO.get(os.path.join(load_path, "%s/instance_segment.png" % name)) # Separate multiple objects into multiple point clouds mask_vals = np.unique(mask[:, :, 2])[ 1:] # each object is indicated by a distinct value in RED channel maxdis = [] skip_count = 0 for i in range(len(mask_vals)): mask_i = np.array(mask[:, :, 2] == mask_vals[i], dtype=np.float32) if plot: cv2.imshow("%s idx:%d" % (name, i), mask_i) cv2.waitKey(0) mask_pcd = deproject(mask_i, inv_k).sum(axis=1) > 0 # opaque_depth = IO.get(os.path.join(load_path, "%s/detph_GroundTruth.exr" % name)) # opaque_pcd = deproject(opaque_depth, inv_k)[mask_pcd] opaque_pcd = IO.get( os.path.join(load_path, "%s/Ground_Truth_%d.pcd" % (name, i))) transp_depth = IO.get(os.path.join(load_path, "%s/depth.exr" % name)) transp_pcd = deproject(transp_depth, inv_k)[mask_pcd] center = transp_pcd.mean(axis=0) if centering: opaque_pcd -= center transp_pcd -= center maxdis.append( np.max((np.max(np.abs(opaque_pcd)), np.max(np.abs(transp_pcd))))) if normalize and not skip: opaque_pcd /= maxdis[-1] * 1.01 transp_pcd /= maxdis[-1] * 1.01 if maxdis[-1] >= 1 and skip: skip_count += 1 continue if not os.path.exists(os.path.join(save_path, name)): os.mkdir(os.path.join(save_path, name)) if maxdis[-1] == 0: print((name, i)) # save centering and scaling factors factors = { 'centering': centering, 'center_position': center.tolist(), 'normalize': normalize, 'normalize_factor': maxdis[-1] * 1.01, } with open( os.path.join(save_path, "%s/scale_factor_%d.json" % (name, i)), 'w') as outfile: json.dump(factors, outfile) # IO.put(os.path.join(save_path, "%s/depth2pcd_GT_%d.pcd" % (name, i)), opaque_pcd) IO.put( os.path.join(save_path, "%s/Ground_Truth_recenter_%d.pcd" % (name, i)), opaque_pcd) IO.put(os.path.join(save_path, "%s/depth2pcd_%d.pcd" % (name, i)), transp_pcd) # print(mask_pcd.sum()) # print(opaque_pcd.shape) # print(transp_pcd.shape) return np.max(maxdis), skip_count