def display_sample(rgb_obs, semantic_obs=np.array([]), depth_obs=np.array([])): from habitat_sim.utils.common import d3_40_colors_rgb rgb_img = Image.fromarray(rgb_obs, mode="RGBA") arr = [rgb_img] titles = ["rgb"] if semantic_obs.size != 0: semantic_img = Image.new("P", (semantic_obs.shape[1], semantic_obs.shape[0])) semantic_img.putpalette(d3_40_colors_rgb.flatten()) semantic_img.putdata((semantic_obs.flatten() % 40).astype(np.uint8)) semantic_img = semantic_img.convert("RGBA") arr.append(semantic_img) titles.append("semantic") if depth_obs.size != 0: depth_img = Image.fromarray((depth_obs / 10 * 255).astype(np.uint8), mode="L") arr.append(depth_img) titles.append("depth") plt.figure(figsize=(12, 8)) for i, data in enumerate(arr): ax = plt.subplot(1, 3, i + 1) ax.axis("off") ax.set_title(titles[i]) plt.imshow(data) plt.show(block=False)
def display_sample(rgb_obs, semantic_obs, depth_obs, visualize=False): rgb_img = Image.fromarray(rgb_obs, mode="RGBA") semantic_img = Image.new("P", (semantic_obs.shape[1], semantic_obs.shape[0])) semantic_img.putpalette(d3_40_colors_rgb.flatten()) semantic_img.putdata((semantic_obs.flatten() % 40).astype(np.uint8)) semantic_img = semantic_img.convert("RGBA") # st() depth_img = Image.fromarray((depth_obs / 10 * 255).astype(np.uint8), mode="L") display_img = cv2.cvtColor(np.asarray(rgb_img), cv2.COLOR_RGB2BGR) #display_img = cv2. cv2.imshow('img', display_img) if visualize: arr = [rgb_img, semantic_img, depth_img] titles = ['rgb', 'semantic', 'depth'] plt.figure(figsize=(12, 8)) for i, data in enumerate(arr): ax = plt.subplot(1, 3, i + 1) ax.axis('off') ax.set_title(titles[i]) plt.imshow(data) # plt.pause() plt.show() plt.pause(0.5) # cv2.imshow() plt.close()
def display_sample2(rgb_obs, depth_obs, semantic_obs, savepath=None): depth_obs = depth_obs / np.amax(depth_obs) # normalize for visualization rgb_img = Image.fromarray(rgb_obs, mode="RGB") depth_img = Image.fromarray((depth_obs * 255).astype(np.uint8), mode="L") semantic_img = Image.new("P", (semantic_obs.shape[1], semantic_obs.shape[0])) semantic_img.putpalette(d3_40_colors_rgb.flatten()) semantic_img.putdata((semantic_obs.flatten() % 40).astype(np.uint8)) semantic_img = semantic_img.convert("RGBA") arr = [rgb_img, depth_img, semantic_img] titles = ['rgb', 'depth', 'sseg'] plt.figure(figsize=(12, 8)) for i, data in enumerate(arr): ax = plt.subplot(1, 3, i + 1) ax.axis('off') ax.set_title(titles[i]) plt.imshow(data) if savepath is None: plt.show() else: plt.savefig(savepath, bbox_inches='tight', pad_inches=0, dpi=100) plt.close()
def save_semantic_observation(self, obs, total_frames): semantic_obs = obs["semantic_sensor"] semantic_img = Image.new( "P", (semantic_obs.shape[1], semantic_obs.shape[0])) semantic_img.putpalette(d3_40_colors_rgb.flatten()) semantic_img.putdata((semantic_obs.flatten() % 40).astype(np.uint8)) semantic_img.save("test.sem.%05d.png" % total_frames)
def display_sample(observations): """Plot RGB, Semantic and Depth images""" rgb_obs = observations["color_sensor"] semantic_obs = observations["semantic_sensor"] depth_obs = observations["depth_sensor"] rgb_img = Image.fromarray(rgb_obs, mode="RGBA") semantic_img = Image.new("P", (semantic_obs.shape[1], semantic_obs.shape[0])) semantic_img.putpalette(d3_40_colors_rgb.flatten()) semantic_img.putdata((semantic_obs.flatten() % 40).astype(np.uint8)) semantic_img = semantic_img.convert("RGBA") depth_img = Image.fromarray((depth_obs / 10 * 255).astype(np.uint8), mode="L") arr = [rgb_img, semantic_img, depth_img] titles = ['rgb', 'semantic', 'depth'] plt.figure(figsize=(12, 8)) for i, data in enumerate(arr): ax = plt.subplot(1, 3, i + 1) ax.axis('off') ax.set_title(titles[i]) plt.imshow(data) plt.show()
def get_semantic_image(semantic_labels): semantic_img = Image.new( "P", (semantic_labels.shape[1], semantic_labels.shape[0])) semantic_img.putpalette(d3_40_colors_rgb.flatten()) semantic_img.putdata((semantic_labels.flatten() % 40).astype(np.uint8)) semantic_img = semantic_img.convert("RGB") return semantic_img
def color_semantic_image(semantic_obs, to_numpy=True): semantic_img = Image.new("P", (semantic_obs.shape[1], semantic_obs.shape[0])) semantic_img.putpalette(d3_40_colors_rgb.flatten()) semantic_img.putdata((semantic_obs.flatten() % 40).astype(np.uint8)) semantic_img = semantic_img.convert("RGBA") if to_numpy: semantic_img = np.array(semantic_img)[:, :, :3] return semantic_img
def save_semantic_observation(self, obs, total_frames): semantic_obs = obs["semantic_sensor"] semantic_img = Image.new( "P", (semantic_obs.shape[1], semantic_obs.shape[0])) semantic_img.putpalette(d3_40_colors_rgb.flatten()) semantic_img.putdata((semantic_obs.flatten() % 40).astype(np.uint8)) if self._demo_type == DemoRunnerType.AB_TEST: if self._group_id == ABTestGroup.CONTROL: semantic_img.save("test.sem.control.%05d.png" % total_frames) else: semantic_img.save("test.sem.test.%05d.png" % total_frames) else: semantic_img.save("test.sem.%05d.png" % total_frames)
def semantic_to_rgb(semantic_image: np.ndarray) -> np.ndarray: """Map semantic ids to colors and genereate an rgb image :param semantic_image: Raw semantic observation image from sensor output. :return: rgb semantic image data. """ semantic_image_rgb = Image.new( "P", (semantic_image.shape[1], semantic_image.shape[0]) ) semantic_image_rgb.putpalette(d3_40_colors_rgb.flatten()) semantic_image_rgb.putdata((semantic_image.flatten() % 40).astype(np.uint8)) semantic_image_rgb = semantic_image_rgb.convert("RGBA") return semantic_image_rgb
def get_visuals(observations): """Returns PIL versions of RGB, Semantic and Depth images, also returns Depth array""" rgb_img = observations["color_sensor"] rgb_img = Image.fromarray(rgb_img, mode="RGBA") sem = observations["semantic_sensor"] sem_img = Image.new("P", (sem.shape[1], sem.shape[0])) sem_img.putpalette(d3_40_colors_rgb.flatten()) sem_img.putdata((sem.flatten() % 40).astype(np.uint8)) sem_img = sem_img.convert("RGBA") dep_arr = observations["depth_sensor"] dep_img = Image.fromarray((dep_arr / 10 * 255).astype(np.uint8), mode="L") return rgb_img, sem_img, dep_img, dep_arr
def display_sample(self, rgb_obs, semantic_obs, depth_obs, mainobj=None, visualize=False): rgb_img = Image.fromarray(rgb_obs, mode="RGBA") semantic_img = Image.new( "P", (semantic_obs.shape[1], semantic_obs.shape[0])) semantic_img.putpalette(d3_40_colors_rgb.flatten()) semantic_img.putdata((semantic_obs.flatten() % 40).astype(np.uint8)) semantic_img = semantic_img.convert("RGBA") # st() depth_img = Image.fromarray((depth_obs / 10 * 255).astype(np.uint8), mode="L") display_img = cv2.cvtColor(np.asarray(rgb_img), cv2.COLOR_RGB2BGR) #print(display_img.shape) # mask_image = False # if mask_image and mainobj is not None: # main_id = int(mainobj.id[1:]) # print("MAINID ", main_id) # # semantic = observations["semantic_sensor"] # display_img[semantic_obs == main_id] = [1, 0, 1] # st() #display_img = cv2 plt.imshow(display_img) plt.show() # cv2.imshow('img',display_img) if visualize: arr = [rgb_img, semantic_img, depth_img] titles = ['rgb', 'semantic', 'depth'] plt.figure(figsize=(12, 8)) for i, data in enumerate(arr): ax = plt.subplot(1, 3, i + 1) ax.axis('off') ax.set_title(titles[i]) plt.imshow(data) # plt.pause() plt.show()
def display_sample(rgb_obs, semantic_obs, depth_obs, save_file=None): rgb_img = Image.fromarray(rgb_obs, mode="RGBA") semantic_img = Image.new("P", (semantic_obs.shape[1], semantic_obs.shape[0])) semantic_img.putpalette(d3_40_colors_rgb.flatten()) semantic_img.putdata((semantic_obs.flatten() % 40).astype(np.uint8)) semantic_img = semantic_img.convert("RGBA") depth_img = Image.fromarray((depth_obs / 10 * 255).astype(np.uint8), mode="L") arr = [rgb_img, semantic_img, depth_img] titles = ['rgb', 'semantic', 'depth'] plt.figure(figsize=(12, 8)) for i, data in enumerate(arr): ax = plt.subplot(1, 3, i + 1) ax.axis('off') ax.set_title(titles[i]) plt.imshow(data) if not save_file: plt.show() else: plt.savefig(save_file)
def display_sample(self, rgb_obs, semantic_obs, depth_obs, mainobj=None, visualize=False): rgb_img = Image.fromarray(rgb_obs, mode="RGBA") semantic_img = Image.new( "P", (semantic_obs.shape[1], semantic_obs.shape[0])) semantic_img.putpalette(d3_40_colors_rgb.flatten()) semantic_img.putdata((semantic_obs.flatten() % 40).astype(np.uint8)) semantic_img = semantic_img.convert("RGBA") # st() depth_img = Image.fromarray((depth_obs / 10 * 255).astype(np.uint8), mode="L") display_img = cv2.cvtColor(np.asarray(rgb_img), cv2.COLOR_RGB2BGR) #print(display_img.shape) # mask_image = False # if mask_image and mainobj is not None: # main_id = int(mainobj.id[1:]) # print("MAINID ", main_id) # # semantic = observations["semantic_sensor"] # display_img[semantic_obs == main_id] = [1, 0, 1] # st() #display_img = cv2 plt.imshow(display_img) plt.show() im = rgb_img[..., :3] im = im[:, :, ::-1] outputs = self.maskrcnn(im) pred_masks = outputs['instances'].pred_masks pred_boxes = outputs['instances'].pred_boxes.tensor pred_classes = outputs['instances'].pred_classes pred_scores = outputs['instances'].scores # converts instance segmentation to individual masks and bbox # visualisations v = Visualizer(im[:, :, ::-1], MetadataCatalog.get(self.cfg.DATASETS.TRAIN[0]), scale=1.2) out = v.draw_instance_predictions(outputs['instances'].to("cpu")) seg_im = out.get_image() # cv2.imshow('img',display_img) if visualize: arr = [rgb_img, semantic_img, depth_img, seg_im] titles = ['rgb', 'semantic', 'depth', 'seg_im'] plt.figure(figsize=(12, 8)) for i, data in enumerate(arr): ax = plt.subplot(1, 3, i + 1) ax.axis('off') ax.set_title(titles[i]) plt.imshow(data) # plt.pause() plt.show()
def convert_semantic_object_to_rgb(x): semantic_img = Image.new("P", (x.shape[1], x.shape[0])) semantic_img.putpalette(d3_40_colors_rgb.flatten()) semantic_img.putdata((x.flatten() % 40).astype(np.uint8)) semantic_img = np.array(semantic_img.convert("RGB")) return semantic_img