def online_aug_val_test(self, idx):
A_path = self.A_paths[idx]
B_path = self.B_paths[idx]
rois_path = self.rois_paths[idx]
A = cv2.imread(A_path, -1) # [H, W, C] C:bgr
B = np.zeros((A.shape[0], A.shape[1]), dtype=np.float32 )
rois = read_labels(rois_path, ignore_cate=self.ignore_cate_list) # list of instances of class ObjectLabel, see obj_utils.py
raw_boxlist = rois2boxlist(rois, (A.shape[1], A.shape[0]))
flip_flg, resize_size, crop_size, pad, resize_ratio = self.set_flip_pad_reshape_crop(A)
A_crop = np.pad(A, ((pad[0], pad[1]), (pad[2], pad[3]), (0, 0)), 'constant', constant_values=(0, 0))
B_crop = np.pad(B, ((pad[0], pad[1]), (pad[2], pad[3])), 'constant', constant_values=(0, 0))
raw_boxlist.bbox[:, 0::2] += pad[2]
raw_boxlist.bbox[:, 1::2] += pad[0]
boxes = raw_boxlist.bbox
A_crop = A_crop.transpose((2, 0, 1))
B_crop = B_crop[np.newaxis, :, :]
# change the color channel, bgr->rgb
A_crop = A_crop[::-1, :, :]
# to torch, normalize
A_crop = self.scale_torch(A_crop, 255.)
B_crop = self.scale_torch(B_crop, 1.0)
data = {'A': A_crop, 'B': B_crop, 'bbox': boxes,
'A_raw': A, 'B_raw': B, 'A_paths': A_path, 'B_paths': B_path, 'pad_raw': np.array(pad)}
return data
def online_aug_train(self, idx):
A_path = self.A_paths[idx]
B_path = self.B_paths[idx]
rois_path = self.rois_paths[idx]
A = cv2.imread(A_path, -1) # [H, W, C] C:bgr
B = cv2.imread(B_path, -1) / self.depth_normalize #[0.0, 1.0]
#B = np.load(B_path) / (self.depth_normalize / 255.) #[0.0, 1.0]
#B[B<0] = -1
rois = read_labels(
rois_path, ignore_cate=self.ignore_cate_list
) # list of instances of class ObjectLabel, see obj_utils.py
#rois_mask = rois2mask(rois, B.shape) # use a mask which has same shape to lidepth to prepresent the rois. rois have value 1., else 0.
#B_rois = B * rois_mask
flip_flg, resize_size, crop_size, pad, resize_ratio = self.set_flip_pad_reshape_crop(
A)
A_crop = self.flip_pad_reshape_crop(A, flip_flg, resize_size,
crop_size, pad, 128)
B_crop = self.flip_pad_reshape_crop(B, flip_flg, resize_size,
crop_size, pad, -1)
raw_boxlist = rois2boxlist(rois, (A.shape[1], A.shape[0]))
boxes_padding = self.box_flip_pad_reshape_crop(raw_boxlist, flip_flg,
resize_size, crop_size,
pad)
B_rois_crop = np.zeros_like(B_crop)
rois_mask_crop = np.zeros_like(B_crop)
boxes = boxes_padding
num_boxes = boxes.size(0)
for idx_box in range(num_boxes):
box = boxes[idx_box]
if box.sum() == 0:
continue
x1, y1, x2, y2 = map(int, box)
B_rois_crop[y1:y2, x1:x2] = B_crop[y1:y2, x1:x2]
rois_mask_crop[y1:y2, x1:x2] = 1
A_crop = A_crop.transpose((2, 0, 1))
B_crop = B_crop[np.newaxis, :, :]
B_rois_crop = B_rois_crop[np.newaxis, :, :]
rois_mask_crop = rois_mask_crop[np.newaxis, :, :]
# change the color channel, bgr->rgb
A_crop = A_crop[::-1, :, :]
# to torch, normalize
A_crop = self.scale_torch(A_crop, 255.)
B_crop = self.scale_torch(B_crop, resize_ratio)
B_rois_crop = self.scale_torch(B_rois_crop, resize_ratio)
rois_mask_crop = np.floor(rois_mask_crop).astype(np.uint8)
rois_mask_crop = torch.from_numpy(rois_mask_crop.copy())
B_classes = self.depth_to_class(B_crop)
B_rois_classes = self.depth_to_class(B_rois_crop)
invalid_side = [0, 0, 0, 0] if crop_size[1] != 0 else [
int((pad[0] + 50) * resize_ratio), 0, 0, 0
]
A = np.pad(A, ((pad[0], pad[1]), (pad[2], pad[3]), (0, 0)),
'constant',
constant_values=(0, 0))
B = np.pad(B, ((pad[0], pad[1]), (pad[2], pad[3])),
'constant',
constant_values=(0, 0))
data = {
'A': A_crop,
'B': B_crop,
'A_raw': A,
'B_raw': B,
'B_classes': B_classes,
'B_rois_classes': B_rois_classes,
'A_paths': A_path,
'B_paths': B_path,
'invalid_side': np.array(invalid_side),
'pad_raw': np.array(pad),
'rois_mask': rois_mask_crop,
'bbox': boxes_padding
}
return data
def online_aug_val_test(self, idx):
A_path = self.A_paths[idx]
B_path = self.B_paths[idx]
rois_path = self.rois_paths[idx]
A = cv2.imread(A_path, -1) # [H, W, C] C:bgr
B = cv2.imread(B_path, -1) / (self.depth_normalize) #[0.0, 1.0]
#B = np.load(B_path) / (self.depth_normalize / 255.) #[0.0, 1.0]
#B[B<0] = 0
rois = read_labels(
rois_path, ignore_cate=self.ignore_cate_list
) # list of instances of class ObjectLabel, see obj_utils.py
raw_boxlist = rois2boxlist(rois, (A.shape[1], A.shape[0]))
#rois_mask = rois2mask(rois, B.shape) # use a mask which has same shape to lidepth to prepresent the rois. rois have value 1., else 0.
rois_mask = np.zeros_like(B)
B_rois = np.zeros_like(B)
B_bg = np.copy(B)
boxes = raw_boxlist.bbox
num_boxes = boxes.size(0)
for idx_box in range(num_boxes):
box = boxes[idx_box]
x1, y1, x2, y2 = map(int, box)
B_rois[y1:y2, x1:x2] = B[y1:y2, x1:x2]
B_bg[y1:y2, x1:x2] = 0.
rois_mask[y1:y2, x1:x2] = 1
flip_flg, resize_size, crop_size, pad, resize_ratio = self.set_flip_pad_reshape_crop(
A)
crop_size_l = [pad[2], 0, cfg.CROP_SIZE[1], cfg.CROP_SIZE[0]]
crop_size_m = [
cfg.CROP_SIZE[1] + pad[2] - 20, 0, cfg.CROP_SIZE[1],
cfg.CROP_SIZE[0]
]
crop_size_r = [
self.uniform_size[1] - cfg.CROP_SIZE[1], 0, cfg.CROP_SIZE[1],
cfg.CROP_SIZE[0]
]
A_crop_l = self.flip_pad_reshape_crop(A, flip_flg, resize_size,
crop_size_l, pad, 128)
A_crop_l = A_crop_l.transpose((2, 0, 1))
A_crop_l = A_crop_l[::-1, :, :]
boxes_l = self.box_flip_pad_reshape_crop(raw_boxlist, flip_flg,
resize_size, crop_size_l, pad)
rois_mask_crop_l = self.flip_pad_reshape_crop(rois_mask, flip_flg,
resize_size, crop_size_l,
pad, 0)
rois_mask_crop_l = rois_mask_crop_l[np.newaxis, :, :]
A_crop_m = self.flip_pad_reshape_crop(A, flip_flg, resize_size,
crop_size_m, pad, 128)
A_crop_m = A_crop_m.transpose((2, 0, 1))
A_crop_m = A_crop_m[::-1, :, :]
boxes_m = self.box_flip_pad_reshape_crop(raw_boxlist, flip_flg,
resize_size, crop_size_m, pad)
rois_mask_crop_m = self.flip_pad_reshape_crop(rois_mask, flip_flg,
resize_size, crop_size_m,
pad, 0)
rois_mask_crop_m = rois_mask_crop_m[np.newaxis, :, :]
A_crop_r = self.flip_pad_reshape_crop(A, flip_flg, resize_size,
crop_size_r, pad, 128)
A_crop_r = A_crop_r.transpose((2, 0, 1))
A_crop_r = A_crop_r[::-1, :, :]
boxes_r = self.box_flip_pad_reshape_crop(raw_boxlist, flip_flg,
resize_size, crop_size_r, pad)
rois_mask_crop_r = self.flip_pad_reshape_crop(rois_mask, flip_flg,
resize_size, crop_size_r,
pad, 0)
rois_mask_crop_r = rois_mask_crop_r[np.newaxis, :, :]
A_crop_l = self.scale_torch(A_crop_l, 255.)
rois_mask_crop_l = np.floor(rois_mask_crop_l).astype(np.uint8)
rois_mask_crop_l = torch.from_numpy(rois_mask_crop_l.copy())
A_crop_m = self.scale_torch(A_crop_m, 255.)
rois_mask_crop_m = np.floor(rois_mask_crop_m).astype(np.uint8)
rois_mask_crop_m = torch.from_numpy(rois_mask_crop_m.copy())
A_crop_r = self.scale_torch(A_crop_r, 255.)
rois_mask_crop_r = np.floor(rois_mask_crop_r).astype(np.uint8)
rois_mask_crop_r = torch.from_numpy(rois_mask_crop_r.copy())
A = np.pad(A, ((pad[0], pad[1]), (pad[2], pad[3]), (0, 0)),
'constant',
constant_values=(0, 0))
B = np.pad(B, ((pad[0], pad[1]), (pad[2], pad[3])),
'constant',
constant_values=(0, 0))
B_rois = np.pad(B_rois, ((pad[0], pad[1]), (pad[2], pad[3])),
'constant',
constant_values=(0, 0))
B_bg = np.pad(B_bg, ((pad[0], pad[1]), (pad[2], pad[3])),
'constant',
constant_values=(0, 0))
crop_lmr = np.array((crop_size_l, crop_size_m, crop_size_r))
A_crop = A.transpose((2, 0, 1))
B_crop = B[np.newaxis, :, :]
# change the color channel, bgr->rgb
A_crop = A_crop[::-1, :, :]
# to torch, normalize
A_crop = self.scale_torch(A_crop, 255.)
B_crop = self.scale_torch(B_crop, 1.0)
data = {
'A': A_crop,
'B': B_crop,
'A_l': A_crop_l,
'A_m': A_crop_m,
'A_r': A_crop_r,
'rois_mask_l': rois_mask_crop_l,
'rois_mask_m': rois_mask_crop_m,
'rois_mask_r': rois_mask_crop_r,
'bbox_l': boxes_l,
'bbox_m': boxes_m,
'bbox_r': boxes_r,
'A_raw': A,
'B_raw': B,
'B_raw_rois': B_rois,
'B_raw_bg': B_bg,
'A_paths': A_path,
'B_paths': B_path,
'pad_raw': np.array(pad),
'crop_lmr': crop_lmr
}
return data
def online_aug_train(self, idx):
A_path = self.A_paths[idx]
B_path = self.B_paths[idx]
rois_path = self.rois_paths[idx]
A = cv2.imread(A_path, -1) # [H, W, C] C:bgr
B = cv2.imread(B_path, -1) / self.depth_normalize #[0.0, 1.0]
#B = np.load(B_path) / (self.depth_normalize / 255.) #[0.0, 1.0]
#B[B<0] = -1
rois = read_labels(
rois_path, ignore_cate=self.ignore_cate_list
) # list of instances of class ObjectLabel, see obj_utils.py
rois_mask = rois2mask(
rois, B.shape
) # use a mask which has same shape to lidepth to prepresent the rois. rois have value 1., else 0.
B_rois = B * rois_mask
flip_flg, resize_size, crop_size, pad, resize_ratio = self.set_flip_pad_reshape_crop(
A)
A_crop = self.flip_pad_reshape_crop(A, flip_flg, resize_size,
crop_size, pad, 128)
B_crop = self.flip_pad_reshape_crop(B, flip_flg, resize_size,
crop_size, pad, -1)
B_rois_crop = self.flip_pad_reshape_crop(B_rois, flip_flg, resize_size,
crop_size, pad, -1)
rois_mask_crop = self.flip_pad_reshape_crop(rois_mask, flip_flg,
resize_size, crop_size,
pad, 0)
A_crop = A_crop.transpose((2, 0, 1))
B_crop = B_crop[np.newaxis, :, :]
B_rois_crop = B_rois_crop[np.newaxis, :, :]
rois_mask_crop = rois_mask_crop[np.newaxis, :, :]
# change the color channel, bgr->rgb
A_crop = A_crop[::-1, :, :]
# to torch, normalize
A_crop = self.scale_torch(A_crop, 255.)
B_crop = self.scale_torch(B_crop, resize_ratio)
B_rois_crop = self.scale_torch(B_rois_crop, resize_ratio)
rois_mask_crop = np.ceil(rois_mask_crop).astype(np.uint8)
rois_mask_crop = torch.from_numpy(rois_mask_crop.copy())
B_classes = self.depth_to_class(B_crop)
B_rois_classes = self.depth_to_class(B_rois_crop)
invalid_side = [0, 0, 0, 0] if crop_size[1] != 0 else [
int((pad[0] + 50) * resize_ratio), 0, 0, 0
]
A = np.pad(A, ((pad[0], pad[1]), (pad[2], pad[3]), (0, 0)),
'constant',
constant_values=(0, 0))
B = np.pad(B, ((pad[0], pad[1]), (pad[2], pad[3])),
'constant',
constant_values=(0, 0))
data = {
'A': A_crop,
'B': B_crop,
'A_raw': A,
'B_raw': B,
'B_classes': B_classes,
'B_rois_classes': B_rois_classes,
'A_paths': A_path,
'B_paths': B_path,
'invalid_side': np.array(invalid_side),
'pad_raw': np.array(pad),
'rois_mask': rois_mask_crop
}
return data
def online_aug_val_test(self, idx):
A_path = self.A_paths[idx]
B_path = self.B_paths[idx]
rois_path = self.rois_paths[idx]
A = cv2.imread(A_path, -1) # [H, W, C] C:bgr
B = cv2.imread(B_path, -1) / (self.depth_normalize) #[0.0, 1.0]
#B = np.load(B_path) / (self.depth_normalize / 255.) #[0.0, 1.0]
#B[B<0] = 0
rois = read_labels(
rois_path, ignore_cate=self.ignore_cate_list
) # list of instances of class ObjectLabel, see obj_utils.py
rois_mask = rois2mask_shrink(
rois, B.shape
) # use a mask which has same shape to lidepth to prepresent the rois. rois have value 1., else 0.
B_rois = B * rois_mask
flip_flg, resize_size, crop_size, pad, resize_ratio = self.set_flip_pad_reshape_crop(
A)
crop_size_l = [pad[2], 0, cfg.CROP_SIZE[1], cfg.CROP_SIZE[0]]
crop_size_m = [
cfg.CROP_SIZE[1] + pad[2] - 20, 0, cfg.CROP_SIZE[1],
cfg.CROP_SIZE[0]
]
crop_size_r = [
self.uniform_size[1] - cfg.CROP_SIZE[1], 0, cfg.CROP_SIZE[1],
cfg.CROP_SIZE[0]
]
A_crop_l = self.flip_pad_reshape_crop(A, flip_flg, resize_size,
crop_size_l, pad, 128)
A_crop_l = A_crop_l.transpose((2, 0, 1))
A_crop_l = A_crop_l[::-1, :, :]
A_crop_m = self.flip_pad_reshape_crop(A, flip_flg, resize_size,
crop_size_m, pad, 128)
A_crop_m = A_crop_m.transpose((2, 0, 1))
A_crop_m = A_crop_m[::-1, :, :]
A_crop_r = self.flip_pad_reshape_crop(A, flip_flg, resize_size,
crop_size_r, pad, 128)
A_crop_r = A_crop_r.transpose((2, 0, 1))
A_crop_r = A_crop_r[::-1, :, :]
A_crop_l = self.scale_torch(A_crop_l, 255.)
A_crop_m = self.scale_torch(A_crop_m, 255.)
A_crop_r = self.scale_torch(A_crop_r, 255.)
A = np.pad(A, ((pad[0], pad[1]), (pad[2], pad[3]), (0, 0)),
'constant',
constant_values=(0, 0))
B = np.pad(B, ((pad[0], pad[1]), (pad[2], pad[3])),
'constant',
constant_values=(0, 0))
B_rois = np.pad(B_rois, ((pad[0], pad[1]), (pad[2], pad[3])),
'constant',
constant_values=(0, 0))
crop_lmr = np.array((crop_size_l, crop_size_m, crop_size_r))
A_crop = A.transpose((2, 0, 1))
B_crop = B[np.newaxis, :, :]
# change the color channel, bgr->rgb
A_crop = A_crop[::-1, :, :]
# to torch, normalize
A_crop = self.scale_torch(A_crop, 255.)
B_crop = self.scale_torch(B_crop, 1.0)
data = {
'A': A_crop,
'B': B_crop,
'A_l': A_crop_l,
'A_m': A_crop_m,
'A_r': A_crop_r,
'A_raw': A,
'B_raw': B,
'B_raw_rois': B_rois,
'A_paths': A_path,
'B_paths': B_path,
'pad_raw': np.array(pad),
'crop_lmr': crop_lmr
}
return data