def online_aug_val_test(self, idx): A_path = self.A_paths[idx] B_path = self.B_paths[idx] rois_path = self.rois_paths[idx] A = cv2.imread(A_path, -1) # [H, W, C] C:bgr B = np.zeros((A.shape[0], A.shape[1]), dtype=np.float32 ) rois = read_labels(rois_path, ignore_cate=self.ignore_cate_list) # list of instances of class ObjectLabel, see obj_utils.py raw_boxlist = rois2boxlist(rois, (A.shape[1], A.shape[0])) flip_flg, resize_size, crop_size, pad, resize_ratio = self.set_flip_pad_reshape_crop(A) A_crop = np.pad(A, ((pad[0], pad[1]), (pad[2], pad[3]), (0, 0)), 'constant', constant_values=(0, 0)) B_crop = np.pad(B, ((pad[0], pad[1]), (pad[2], pad[3])), 'constant', constant_values=(0, 0)) raw_boxlist.bbox[:, 0::2] += pad[2] raw_boxlist.bbox[:, 1::2] += pad[0] boxes = raw_boxlist.bbox A_crop = A_crop.transpose((2, 0, 1)) B_crop = B_crop[np.newaxis, :, :] # change the color channel, bgr->rgb A_crop = A_crop[::-1, :, :] # to torch, normalize A_crop = self.scale_torch(A_crop, 255.) B_crop = self.scale_torch(B_crop, 1.0) data = {'A': A_crop, 'B': B_crop, 'bbox': boxes, 'A_raw': A, 'B_raw': B, 'A_paths': A_path, 'B_paths': B_path, 'pad_raw': np.array(pad)} return data
def online_aug_train(self, idx): A_path = self.A_paths[idx] B_path = self.B_paths[idx] rois_path = self.rois_paths[idx] A = cv2.imread(A_path, -1) # [H, W, C] C:bgr B = cv2.imread(B_path, -1) / self.depth_normalize #[0.0, 1.0] #B = np.load(B_path) / (self.depth_normalize / 255.) #[0.0, 1.0] #B[B<0] = -1 rois = read_labels( rois_path, ignore_cate=self.ignore_cate_list ) # list of instances of class ObjectLabel, see obj_utils.py #rois_mask = rois2mask(rois, B.shape) # use a mask which has same shape to lidepth to prepresent the rois. rois have value 1., else 0. #B_rois = B * rois_mask flip_flg, resize_size, crop_size, pad, resize_ratio = self.set_flip_pad_reshape_crop( A) A_crop = self.flip_pad_reshape_crop(A, flip_flg, resize_size, crop_size, pad, 128) B_crop = self.flip_pad_reshape_crop(B, flip_flg, resize_size, crop_size, pad, -1) raw_boxlist = rois2boxlist(rois, (A.shape[1], A.shape[0])) boxes_padding = self.box_flip_pad_reshape_crop(raw_boxlist, flip_flg, resize_size, crop_size, pad) B_rois_crop = np.zeros_like(B_crop) rois_mask_crop = np.zeros_like(B_crop) boxes = boxes_padding num_boxes = boxes.size(0) for idx_box in range(num_boxes): box = boxes[idx_box] if box.sum() == 0: continue x1, y1, x2, y2 = map(int, box) B_rois_crop[y1:y2, x1:x2] = B_crop[y1:y2, x1:x2] rois_mask_crop[y1:y2, x1:x2] = 1 A_crop = A_crop.transpose((2, 0, 1)) B_crop = B_crop[np.newaxis, :, :] B_rois_crop = B_rois_crop[np.newaxis, :, :] rois_mask_crop = rois_mask_crop[np.newaxis, :, :] # change the color channel, bgr->rgb A_crop = A_crop[::-1, :, :] # to torch, normalize A_crop = self.scale_torch(A_crop, 255.) B_crop = self.scale_torch(B_crop, resize_ratio) B_rois_crop = self.scale_torch(B_rois_crop, resize_ratio) rois_mask_crop = np.floor(rois_mask_crop).astype(np.uint8) rois_mask_crop = torch.from_numpy(rois_mask_crop.copy()) B_classes = self.depth_to_class(B_crop) B_rois_classes = self.depth_to_class(B_rois_crop) invalid_side = [0, 0, 0, 0] if crop_size[1] != 0 else [ int((pad[0] + 50) * resize_ratio), 0, 0, 0 ] A = np.pad(A, ((pad[0], pad[1]), (pad[2], pad[3]), (0, 0)), 'constant', constant_values=(0, 0)) B = np.pad(B, ((pad[0], pad[1]), (pad[2], pad[3])), 'constant', constant_values=(0, 0)) data = { 'A': A_crop, 'B': B_crop, 'A_raw': A, 'B_raw': B, 'B_classes': B_classes, 'B_rois_classes': B_rois_classes, 'A_paths': A_path, 'B_paths': B_path, 'invalid_side': np.array(invalid_side), 'pad_raw': np.array(pad), 'rois_mask': rois_mask_crop, 'bbox': boxes_padding } return data
def online_aug_val_test(self, idx): A_path = self.A_paths[idx] B_path = self.B_paths[idx] rois_path = self.rois_paths[idx] A = cv2.imread(A_path, -1) # [H, W, C] C:bgr B = cv2.imread(B_path, -1) / (self.depth_normalize) #[0.0, 1.0] #B = np.load(B_path) / (self.depth_normalize / 255.) #[0.0, 1.0] #B[B<0] = 0 rois = read_labels( rois_path, ignore_cate=self.ignore_cate_list ) # list of instances of class ObjectLabel, see obj_utils.py raw_boxlist = rois2boxlist(rois, (A.shape[1], A.shape[0])) #rois_mask = rois2mask(rois, B.shape) # use a mask which has same shape to lidepth to prepresent the rois. rois have value 1., else 0. rois_mask = np.zeros_like(B) B_rois = np.zeros_like(B) B_bg = np.copy(B) boxes = raw_boxlist.bbox num_boxes = boxes.size(0) for idx_box in range(num_boxes): box = boxes[idx_box] x1, y1, x2, y2 = map(int, box) B_rois[y1:y2, x1:x2] = B[y1:y2, x1:x2] B_bg[y1:y2, x1:x2] = 0. rois_mask[y1:y2, x1:x2] = 1 flip_flg, resize_size, crop_size, pad, resize_ratio = self.set_flip_pad_reshape_crop( A) crop_size_l = [pad[2], 0, cfg.CROP_SIZE[1], cfg.CROP_SIZE[0]] crop_size_m = [ cfg.CROP_SIZE[1] + pad[2] - 20, 0, cfg.CROP_SIZE[1], cfg.CROP_SIZE[0] ] crop_size_r = [ self.uniform_size[1] - cfg.CROP_SIZE[1], 0, cfg.CROP_SIZE[1], cfg.CROP_SIZE[0] ] A_crop_l = self.flip_pad_reshape_crop(A, flip_flg, resize_size, crop_size_l, pad, 128) A_crop_l = A_crop_l.transpose((2, 0, 1)) A_crop_l = A_crop_l[::-1, :, :] boxes_l = self.box_flip_pad_reshape_crop(raw_boxlist, flip_flg, resize_size, crop_size_l, pad) rois_mask_crop_l = self.flip_pad_reshape_crop(rois_mask, flip_flg, resize_size, crop_size_l, pad, 0) rois_mask_crop_l = rois_mask_crop_l[np.newaxis, :, :] A_crop_m = self.flip_pad_reshape_crop(A, flip_flg, resize_size, crop_size_m, pad, 128) A_crop_m = A_crop_m.transpose((2, 0, 1)) A_crop_m = A_crop_m[::-1, :, :] boxes_m = self.box_flip_pad_reshape_crop(raw_boxlist, flip_flg, resize_size, crop_size_m, pad) rois_mask_crop_m = self.flip_pad_reshape_crop(rois_mask, flip_flg, resize_size, crop_size_m, pad, 0) rois_mask_crop_m = rois_mask_crop_m[np.newaxis, :, :] A_crop_r = self.flip_pad_reshape_crop(A, flip_flg, resize_size, crop_size_r, pad, 128) A_crop_r = A_crop_r.transpose((2, 0, 1)) A_crop_r = A_crop_r[::-1, :, :] boxes_r = self.box_flip_pad_reshape_crop(raw_boxlist, flip_flg, resize_size, crop_size_r, pad) rois_mask_crop_r = self.flip_pad_reshape_crop(rois_mask, flip_flg, resize_size, crop_size_r, pad, 0) rois_mask_crop_r = rois_mask_crop_r[np.newaxis, :, :] A_crop_l = self.scale_torch(A_crop_l, 255.) rois_mask_crop_l = np.floor(rois_mask_crop_l).astype(np.uint8) rois_mask_crop_l = torch.from_numpy(rois_mask_crop_l.copy()) A_crop_m = self.scale_torch(A_crop_m, 255.) rois_mask_crop_m = np.floor(rois_mask_crop_m).astype(np.uint8) rois_mask_crop_m = torch.from_numpy(rois_mask_crop_m.copy()) A_crop_r = self.scale_torch(A_crop_r, 255.) rois_mask_crop_r = np.floor(rois_mask_crop_r).astype(np.uint8) rois_mask_crop_r = torch.from_numpy(rois_mask_crop_r.copy()) A = np.pad(A, ((pad[0], pad[1]), (pad[2], pad[3]), (0, 0)), 'constant', constant_values=(0, 0)) B = np.pad(B, ((pad[0], pad[1]), (pad[2], pad[3])), 'constant', constant_values=(0, 0)) B_rois = np.pad(B_rois, ((pad[0], pad[1]), (pad[2], pad[3])), 'constant', constant_values=(0, 0)) B_bg = np.pad(B_bg, ((pad[0], pad[1]), (pad[2], pad[3])), 'constant', constant_values=(0, 0)) crop_lmr = np.array((crop_size_l, crop_size_m, crop_size_r)) A_crop = A.transpose((2, 0, 1)) B_crop = B[np.newaxis, :, :] # change the color channel, bgr->rgb A_crop = A_crop[::-1, :, :] # to torch, normalize A_crop = self.scale_torch(A_crop, 255.) B_crop = self.scale_torch(B_crop, 1.0) data = { 'A': A_crop, 'B': B_crop, 'A_l': A_crop_l, 'A_m': A_crop_m, 'A_r': A_crop_r, 'rois_mask_l': rois_mask_crop_l, 'rois_mask_m': rois_mask_crop_m, 'rois_mask_r': rois_mask_crop_r, 'bbox_l': boxes_l, 'bbox_m': boxes_m, 'bbox_r': boxes_r, 'A_raw': A, 'B_raw': B, 'B_raw_rois': B_rois, 'B_raw_bg': B_bg, 'A_paths': A_path, 'B_paths': B_path, 'pad_raw': np.array(pad), 'crop_lmr': crop_lmr } return data
def online_aug_train(self, idx): A_path = self.A_paths[idx] B_path = self.B_paths[idx] rois_path = self.rois_paths[idx] A = cv2.imread(A_path, -1) # [H, W, C] C:bgr B = cv2.imread(B_path, -1) / self.depth_normalize #[0.0, 1.0] #B = np.load(B_path) / (self.depth_normalize / 255.) #[0.0, 1.0] #B[B<0] = -1 rois = read_labels( rois_path, ignore_cate=self.ignore_cate_list ) # list of instances of class ObjectLabel, see obj_utils.py rois_mask = rois2mask( rois, B.shape ) # use a mask which has same shape to lidepth to prepresent the rois. rois have value 1., else 0. B_rois = B * rois_mask flip_flg, resize_size, crop_size, pad, resize_ratio = self.set_flip_pad_reshape_crop( A) A_crop = self.flip_pad_reshape_crop(A, flip_flg, resize_size, crop_size, pad, 128) B_crop = self.flip_pad_reshape_crop(B, flip_flg, resize_size, crop_size, pad, -1) B_rois_crop = self.flip_pad_reshape_crop(B_rois, flip_flg, resize_size, crop_size, pad, -1) rois_mask_crop = self.flip_pad_reshape_crop(rois_mask, flip_flg, resize_size, crop_size, pad, 0) A_crop = A_crop.transpose((2, 0, 1)) B_crop = B_crop[np.newaxis, :, :] B_rois_crop = B_rois_crop[np.newaxis, :, :] rois_mask_crop = rois_mask_crop[np.newaxis, :, :] # change the color channel, bgr->rgb A_crop = A_crop[::-1, :, :] # to torch, normalize A_crop = self.scale_torch(A_crop, 255.) B_crop = self.scale_torch(B_crop, resize_ratio) B_rois_crop = self.scale_torch(B_rois_crop, resize_ratio) rois_mask_crop = np.ceil(rois_mask_crop).astype(np.uint8) rois_mask_crop = torch.from_numpy(rois_mask_crop.copy()) B_classes = self.depth_to_class(B_crop) B_rois_classes = self.depth_to_class(B_rois_crop) invalid_side = [0, 0, 0, 0] if crop_size[1] != 0 else [ int((pad[0] + 50) * resize_ratio), 0, 0, 0 ] A = np.pad(A, ((pad[0], pad[1]), (pad[2], pad[3]), (0, 0)), 'constant', constant_values=(0, 0)) B = np.pad(B, ((pad[0], pad[1]), (pad[2], pad[3])), 'constant', constant_values=(0, 0)) data = { 'A': A_crop, 'B': B_crop, 'A_raw': A, 'B_raw': B, 'B_classes': B_classes, 'B_rois_classes': B_rois_classes, 'A_paths': A_path, 'B_paths': B_path, 'invalid_side': np.array(invalid_side), 'pad_raw': np.array(pad), 'rois_mask': rois_mask_crop } return data
def online_aug_val_test(self, idx): A_path = self.A_paths[idx] B_path = self.B_paths[idx] rois_path = self.rois_paths[idx] A = cv2.imread(A_path, -1) # [H, W, C] C:bgr B = cv2.imread(B_path, -1) / (self.depth_normalize) #[0.0, 1.0] #B = np.load(B_path) / (self.depth_normalize / 255.) #[0.0, 1.0] #B[B<0] = 0 rois = read_labels( rois_path, ignore_cate=self.ignore_cate_list ) # list of instances of class ObjectLabel, see obj_utils.py rois_mask = rois2mask_shrink( rois, B.shape ) # use a mask which has same shape to lidepth to prepresent the rois. rois have value 1., else 0. B_rois = B * rois_mask flip_flg, resize_size, crop_size, pad, resize_ratio = self.set_flip_pad_reshape_crop( A) crop_size_l = [pad[2], 0, cfg.CROP_SIZE[1], cfg.CROP_SIZE[0]] crop_size_m = [ cfg.CROP_SIZE[1] + pad[2] - 20, 0, cfg.CROP_SIZE[1], cfg.CROP_SIZE[0] ] crop_size_r = [ self.uniform_size[1] - cfg.CROP_SIZE[1], 0, cfg.CROP_SIZE[1], cfg.CROP_SIZE[0] ] A_crop_l = self.flip_pad_reshape_crop(A, flip_flg, resize_size, crop_size_l, pad, 128) A_crop_l = A_crop_l.transpose((2, 0, 1)) A_crop_l = A_crop_l[::-1, :, :] A_crop_m = self.flip_pad_reshape_crop(A, flip_flg, resize_size, crop_size_m, pad, 128) A_crop_m = A_crop_m.transpose((2, 0, 1)) A_crop_m = A_crop_m[::-1, :, :] A_crop_r = self.flip_pad_reshape_crop(A, flip_flg, resize_size, crop_size_r, pad, 128) A_crop_r = A_crop_r.transpose((2, 0, 1)) A_crop_r = A_crop_r[::-1, :, :] A_crop_l = self.scale_torch(A_crop_l, 255.) A_crop_m = self.scale_torch(A_crop_m, 255.) A_crop_r = self.scale_torch(A_crop_r, 255.) A = np.pad(A, ((pad[0], pad[1]), (pad[2], pad[3]), (0, 0)), 'constant', constant_values=(0, 0)) B = np.pad(B, ((pad[0], pad[1]), (pad[2], pad[3])), 'constant', constant_values=(0, 0)) B_rois = np.pad(B_rois, ((pad[0], pad[1]), (pad[2], pad[3])), 'constant', constant_values=(0, 0)) crop_lmr = np.array((crop_size_l, crop_size_m, crop_size_r)) A_crop = A.transpose((2, 0, 1)) B_crop = B[np.newaxis, :, :] # change the color channel, bgr->rgb A_crop = A_crop[::-1, :, :] # to torch, normalize A_crop = self.scale_torch(A_crop, 255.) B_crop = self.scale_torch(B_crop, 1.0) data = { 'A': A_crop, 'B': B_crop, 'A_l': A_crop_l, 'A_m': A_crop_m, 'A_r': A_crop_r, 'A_raw': A, 'B_raw': B, 'B_raw_rois': B_rois, 'A_paths': A_path, 'B_paths': B_path, 'pad_raw': np.array(pad), 'crop_lmr': crop_lmr } return data