def test_process_image(self): train_gen = MongoDBGenerator( self.collection_details, self.train_data, batch_size=12, processors=[CenterTrackerProcess(self.params)] ) batch_x, batch_y = train_gen[0] batch_size = len(batch_x["img"]) for i in range(batch_size): # input_data is a list in this case and consist of: # [0]: img current frame # [1]: img prev frame # [3]: single channel heatmap curr_img = batch_x["img"][i] prev_img = batch_x["prev_img"][i] mask_img = to_3channel(batch_y[i], OD_CLASS_MAPPING, 0.1, True) prev_mask_img = to_3channel(batch_x["prev_heatmap"][i], OrderedDict([("obj", (255, 255, 255))]), 0.1, True) cv2.imshow("img", curr_img.astype(np.uint8)) cv2.imshow("prev_img", prev_img.astype(np.uint8)) cv2.imshow("prev_heatmap", prev_mask_img) cv2.imshow("mask", mask_img) cv2.waitKey(0)
def show_semseg(self, inp, y_true, y_pred): inp_img = cv2.cvtColor(inp[0].numpy().astype(np.uint8), cv2.COLOR_BGR2RGB).swapaxes(0, 1) surface_img = pygame.surfarray.make_surface(inp_img) self.display.blit(surface_img, (0, 0)) semseg_true = cv2.cvtColor(to_3channel(y_true[0].numpy(), List(SEMSEG_CLASS_MAPPING.items())), cv2.COLOR_BGR2RGB).swapaxes(0, 1) surface_y_true = pygame.surfarray.make_surface(semseg_true) self.display.blit(surface_y_true, (0, self.params.INPUT_HEIGHT)) semseg_pred = cv2.cvtColor(to_3channel(y_pred[0].numpy(), List(SEMSEG_CLASS_MAPPING.items())), cv2.COLOR_BGR2RGB).swapaxes(0, 1) surface_y_pred = pygame.surfarray.make_surface(semseg_pred) self.display.blit(surface_y_pred, (0, int(self.params.INPUT_HEIGHT + self.params.MASK_HEIGHT))) self.step_counter += 1 if self.step_counter % 2000 == 0: pygame.image.save(self.display, f"{self._storage_path}/train_result_{self.step_counter}.png") pygame.display.flip()
def test_process_image(self): train_gen = MongoDBGenerator( [self.collection_details], [self.train_data], batch_size=30, processors=[ ProcessImages(self.params, start_augmentation=[0, 0], show_debug_img=True) ]) for batch_x, batch_y in train_gen: print("New batch") for i in range(len(batch_x[0])): assert len(batch_x[0]) > 0 img1 = batch_x[0][i] heatmap = np.array( batch_y[i] [:, :, :1]) # needed because otherwise numba makes mimimi heatmap = to_3channel(heatmap, List([("object", (0, 0, 255))]), 0.01, True, False) weights = np.stack([batch_y[i][:, :, -1]] * 3, axis=-1) f, (ax1, ax2, ax3) = plt.subplots(1, 3) ax1.imshow( cv2.cvtColor(batch_x[0][i].astype(np.uint8), cv2.COLOR_BGR2RGB)) ax2.imshow(cv2.cvtColor(heatmap, cv2.COLOR_BGR2RGB)) ax3.imshow(weights) plt.show()
def test_process_image(self): train_gen = MongoDBGenerator( [self.collection_details], [self.td], batch_size=10, processors=[ProcessImages(self.params, [0, 0])], shuffle_data=True ) for batch_x, batch_y in train_gen: print("New batch") for i in range(len(batch_x[0])): assert len(batch_x[0]) > 0 img1 = batch_x[0][i] # [0: centernet_channel]: centernet # [centernet_channels+1]: centernet weights # [centernet_channels+1: semseg_classes]: semseg_masks # [semseg_classes+1]: semseg weights # [-1]: depth semseg_cls_items = List(SEMSEG_CLASS_MAPPING.items()) cn_idx_end = self.params.cn_params.mask_channels() semseg_idx_end = (cn_idx_end+1) + len(semseg_cls_items) cn_heatmap = np.array(batch_y[i][:, :, :cn_idx_end]) cn_weights = np.stack([batch_y[i][:, :, cn_idx_end]]*3, axis=-1) cn_heatmap = to_3channel(cn_heatmap, List([("object", (0, 0, 255))]), 0.01, True, False) semseg_mask = np.array(batch_y[i][:, :, cn_idx_end+1:semseg_idx_end]) semseg_mask = to_3channel(semseg_mask, List(SEMSEG_CLASS_MAPPING.items()), 0.01, True, False) semseg_weights = np.stack([batch_y[i][:, :, semseg_idx_end]]*3, axis=-1) depth_map = np.array(batch_y[i][:, :, -1]) f, ((ax11, ax12), (ax21, ax22), (ax31, ax32)) = plt.subplots(3, 2) ax11.imshow(cv2.cvtColor(batch_x[0][i].astype(np.uint8), cv2.COLOR_BGR2RGB)) ax12.imshow(cv2.cvtColor(cmap_depth(depth_map, vmin=0.1, vmax=255.0), cv2.COLOR_BGR2RGB)) ax21.imshow(cv2.cvtColor(cn_heatmap, cv2.COLOR_BGR2RGB)) ax22.imshow(cn_weights) ax31.imshow(cv2.cvtColor(semseg_mask, cv2.COLOR_BGR2RGB)) ax32.imshow(semseg_weights) plt.show()
def test_process_image(self): train_gen = MongoDBGenerator([self.collection_details], [self.train_data], batch_size=10, processors=[ProcessImages(self.params)]) batch_x, batch_y = train_gen[0] for i, input_data in enumerate(batch_x): assert len(input_data) > 0 cls_items = List(SEMSEG_CLASS_MAPPING.items()) nb_classes = len(cls_items) semseg_mask = np.array( batch_y[i] [:, :, :-1]) # needed because otherwise numba makes mimimi mask_img = to_3channel(semseg_mask, cls_items, threshold=0.999) pos_mask = batch_y[i][:, :, -1] f, (ax1, ax2, ax3) = plt.subplots(1, 3) ax1.imshow( cv2.cvtColor(input_data.astype(np.uint8), cv2.COLOR_BGR2RGB)) ax2.imshow(cv2.cvtColor(mask_img, cv2.COLOR_BGR2RGB)) ax3.imshow(pos_mask, cmap="gray", vmin=0.0, vmax=1.0) plt.show()
decoded_mask = np.frombuffer(doc["mask"], np.uint8) gt_mask = cv2.imdecode(decoded_mask, cv2.IMREAD_COLOR) img, roi = resize_img(img, args.img_width, args.img_height, args.offset_bottom) if is_tf_lite: img_input = img interpreter.set_tensor(input_details[0]['index'], [img_input]) #input_shape = input_details[0]['shape'] start_time = time.time() interpreter.invoke() elapsed_time = time.time() - start_time # The function `get_tensor()` returns a copy of the tensor data. Use `tensor()` in order to get a pointer to the tensor. output_data = interpreter.get_tensor(output_details[0]['index']) semseg_img = to_3channel(output_data[0].astype(np.float64), List(SEMSEG_CLASS_MAPPING.items())) else: img_arr = np.array([img]) start_time = time.time() raw_result = model.predict(img_arr) elapsed_time = time.time() - start_time semseg_img = raw_result[0] semseg_img = to_3channel(semseg_img, List(SEMSEG_CLASS_MAPPING.items()), threshold=0.5) # in case we are sure, there is no care parts (e.g. nuscenes) mask these detections out # semseg_img[np.where((semseg_img==[255, 0, 204]).all(axis=2))] = [0, 0, 0] # # somehow the model likes to put non drivable holes in front of us, mask these out # semseg_cut = semseg_img[182:, 240:400, :]
def show_od(self, inp, y_true, y_pred): heatmap_true = np.array( y_true[0][:, :, :1]) # needed because otherwise numba makes mimimi heatmap_true = to_3channel(heatmap_true, List([("object", (0, 0, 255))]), 0.01, True, False) weights = np.stack([y_true[0][:, :, -1]] * 3, axis=-1) heatmap_pred = np.array(y_pred[0][:, :, :1]) heatmap_pred = to_3channel(heatmap_pred, List([("object", (0, 0, 255))]), 0.01, True, False) # display img show_inp_img = inp[0][0].numpy() show_inp_img = show_inp_img.astype(np.uint8) # get objects from y_pred if self._process_output: roi = Roi() prediction = np.array(y_pred[0]) objects = post_processing.process_2d_output( prediction, roi, self.params, 0.2) for obj in objects: color = list(OD_CLASS_MAPPING.values())[obj["cls_idx"]] # top_center = (int(obj["bottom_center"][0]), int(obj["bottom_center"][1] - obj["center_height"])) # bottom_left = (int(obj["bottom_left"][0]), int(obj["bottom_left"][1])) # bottom_center = (int(obj["bottom_center"][0]), int(obj["bottom_center"][1])) # bottom_right = (int(obj["bottom_right"][0]), int(obj["bottom_right"][1])) # cv2.line(show_inp_img, bottom_left, bottom_center, (0, 255, 0) , 1) # cv2.line(show_inp_img, bottom_center, bottom_right, (0, 255, 0) , 1) # cv2.line(show_inp_img, bottom_center, top_center, (0, 255, 0) , 1) top_left = (int(obj["fullbox"][0]), int(obj["fullbox"][1])) bottom_right = (int(obj["fullbox"][0] + obj["fullbox"][2]), int(obj["fullbox"][1] + obj["fullbox"][3])) cv2.rectangle(show_inp_img, top_left, bottom_right, color, 1) cv2.circle(show_inp_img, (int(obj["center"][0]), int(obj["center"][1])), 2, color, 1) inp_img = cv2.cvtColor(show_inp_img, cv2.COLOR_BGR2RGB).swapaxes(0, 1) surface_img = pygame.surfarray.make_surface(inp_img) self._display.blit(surface_img, (0, 0)) # display heatmap y_pred heatmap_pred = cv2.cvtColor(heatmap_pred, cv2.COLOR_BGR2RGB).swapaxes(0, 1) surface_y_pred = pygame.surfarray.make_surface(heatmap_pred) self._display.blit(surface_y_pred, (0, self.params.INPUT_HEIGHT)) # display heatmap y_true heatmap_true = cv2.cvtColor(heatmap_true, cv2.COLOR_BGR2RGB).swapaxes(0, 1) surface_y_true = pygame.surfarray.make_surface(heatmap_true) self._display.blit( surface_y_true, (0, int(self.params.INPUT_HEIGHT + self.params.MASK_HEIGHT))) self._step_counter += 1 if self._step_counter % 2000 == 0: pygame.image.save( self._display, f"{self._storage_path}/train_result_{self._step_counter}.png") pygame.display.flip()