def train_transform_label(self, rgb, depth, label):
s = np.random.uniform(1.0, 1.5) # random scaling
depth_np = depth # / s
angle = np.random.uniform(-5.0, 5.0) # random rotation degrees
do_flip = np.random.uniform(0.0, 1.0) < 0.5 # random horizontal flip
shift_x = np.random.uniform(-50.0, 50.0)
# perform 1st step of data augmentation
transform = transforms.Compose([
# transforms.Translate(shift_x, 0.0),
transforms.Resize(
150.0 / iheight
), # this is for computational efficiency, since rotation can be slow
# transforms.Rotate(angle),
# transforms.Resize(s),
# transforms.CenterCrop(self.output_size),
transforms.HorizontalFlip(do_flip)
])
label_transform = transforms.Compose([
# transforms.Translate(shift_x / 2.0, 0.0),
# transforms.CenterCrop(self.output_size),
transforms.HorizontalFlip(do_flip)
])
rgb_np = transform(rgb)
rgb_np = self.color_jitter(rgb_np) # random color jittering
rgb_np = np.asfarray(rgb_np, dtype='float') / 255
depth_np = transform(depth_np)
label_np = label_transform(label)
return rgb_np, depth_np, label_np
def train_transform(self, rgb, depth):
s = self.getFocalScale()
if (self.augArgs.varFocus): #Variable focal length simulation
depth_np = depth
else:
depth_np = depth / s #Correct for focal length
if (self.augArgs.varScale): #Variable global scale simulation
scale = self.getDepthGroup()
depth_np = depth_np * scale
angle = np.random.uniform(-5.0, 5.0) # random rotation degrees
do_flip = np.random.uniform(0.0, 1.0) < 0.5 # random horizontal flip
# perform 1st step of data augmentation
transform = transforms.Compose([
transforms.Resize(
250.0 / iheight
), # this is for computational efficiency, since rotation can be slow
transforms.Rotate(angle),
transforms.Resize(s),
transforms.CenterCrop(self.output_size),
transforms.HorizontalFlip(do_flip)
])
rgb_np = transform(rgb)
rgb_np = self.color_jitter(rgb_np) # random color jittering
rgb_np = np.asfarray(rgb_np, dtype='float') / 255
depth_np = transform(depth_np)
return rgb_np, depth_np
def train_transform(rgb, sparse, target, rgb_near, args):
# s = np.random.uniform(1.0, 1.5) # random scaling
# angle = np.random.uniform(-5.0, 5.0) # random rotation degrees
do_flip = np.random.uniform(0.0, 1.0) < 0.5 # random horizontal flip
transform_geometric = transforms.Compose([
# transforms.Rotate(angle),
# transforms.Resize(s),
transforms.BottomCrop((oheight, owidth)),
transforms.HorizontalFlip(do_flip)
])
if sparse is not None:
sparse = transform_geometric(sparse)
target = transform_geometric(target)
if rgb is not None:
brightness = np.random.uniform(max(0, 1 - args.jitter),
1 + args.jitter)
contrast = np.random.uniform(max(0, 1 - args.jitter), 1 + args.jitter)
saturation = np.random.uniform(max(0, 1 - args.jitter),
1 + args.jitter)
transform_rgb = transforms.Compose([
transforms.ColorJitter(brightness, contrast, saturation, 0),
transform_geometric
])
rgb = transform_rgb(rgb)
if rgb_near is not None:
rgb_near = transform_rgb(rgb_near)
# sparse = drop_depth_measurements(sparse, 0.9)
return rgb, sparse, target, rgb_near
def train_transform(self, rgb, depth):
s = np.random.uniform(1.0, 1.5) # random scaling
random_size = (int(s * 224), int(s * 224))
depth_np = depth / s
angle = np.random.uniform(-5.0, 5.0) # random rotation degrees
do_flip = np.random.uniform(0.0, 1.0) < 0.5 # random horizontal flip
# perform 1st step of data augmentation
# transform = torchvision.transforms.Compose([
# torchvision.transforms.Resize(self.output_size), # this is for computational efficiency, since rotation can be slow
# torchvision.transforms.RandomRotation(angle),
# torchvision.transforms.Resize(random_size),
# torchvision.transforms.CenterCrop(self.output_size),
# torchvision.transforms.RandomHorizontalFlip(do_flip)
#])
transform2 = transforms.Compose([
transforms.Resize(
250.0 / iheight
), # this is for computational efficiency, since rotation can be slow
transforms.Rotate(angle),
transforms.Resize(s),
transforms.CenterCrop(self.output_size),
transforms.HorizontalFlip(do_flip)
])
rgb_np = transform2(rgb)
#rgb_n = Image.fromarray(np.uint8(rgb_np * 255))
#rgb_np = self.color_jitter(rgb_n) # random color jittering
rgb_np = np.asfarray(rgb_np, dtype='float') / 255
depth_np = transform2(depth_np)
#depth_np = np.asfarray(depth_np, dtype='float') / 255
return rgb_np, depth_np
def train_transform(self, im, gt):
im = np.array(im).astype(np.float32)
gt = np.array(gt).astype(np.float32)
s = np.random.uniform(1.0, 1.5) # random scaling
angle = np.random.uniform(-5.0, 5.0) # random rotation degrees
do_flip = np.random.uniform(0.0, 1.0) < 0.5 # random horizontal flip
color_jitter = my_transforms.ColorJitter(0.4, 0.4, 0.4)
transform = my_transforms.Compose([
my_transforms.Crop(130, 10, 240, 1200),
my_transforms.Resize(460 / 240, interpolation='bilinear'),
my_transforms.Rotate(angle),
my_transforms.Resize(s),
my_transforms.CenterCrop(self.size),
my_transforms.HorizontalFlip(do_flip)
])
im_ = transform(im)
im_ = color_jitter(im_)
gt_ = transform(gt)
im_ = np.array(im_).astype(np.float32)
gt_ = np.array(gt_).astype(np.float32)
im_ /= 255.0
gt_ /= 100.0 * s
im_ = to_tensor(im_)
gt_ = to_tensor(gt_)
gt_ = gt_.unsqueeze(0)
return im_, gt_
def train_transform(self, rgb, depth):
#s = np.random.uniform(1.0, 1.5) # random scaling
#depth_np = depth / s
s = self.getFocalScale()
if (self.augArgs.varFocus): #Variable focal length simulation
depth_np = depth
else:
depth_np = depth / s #Correct for focal length
if (self.augArgs.varScale): #Variable global scale simulation
scale = self.getDepthGroup()
depth_np = depth_np * scale
angle = np.random.uniform(-5.0, 5.0) # random rotation degrees
do_flip = np.random.uniform(0.0, 1.0) < 0.5 # random horizontal flip
# perform 1st step of data augmentation
transform = transforms.Compose([
transforms.Crop(130, 10, 240, 1200),
transforms.Rotate(angle),
transforms.Resize(s),
transforms.CenterCrop(self.output_size),
transforms.HorizontalFlip(do_flip)
])
rgb_np = transform(rgb)
rgb_np = self.color_jitter(rgb_np) # random color jittering
rgb_np = np.asfarray(rgb_np, dtype='float') / 255
# Scipy affine_transform produced RuntimeError when the depth map was
# given as a 'numpy.ndarray'
depth_np = np.asfarray(depth_np, dtype='float32')
depth_np = transform(depth_np)
return rgb_np, depth_np
def train_transform(self, rgb, depth):
scale = np.random.uniform(low=1, high=1.5)
depth = depth / scale
angle = np.random.uniform(-5.0, 5.0)
should_flip = np.random.uniform(0.0, 1.0) < 0.5
h_offset = int((768 - 228) * np.random.uniform(0.0, 1.0))
v_offset = int((1024 - 304) * np.random.uniform(0.0, 1.0))
base_transform = transforms.Compose([
transforms.Resize(250 / iheight),
transforms.Rotate(angle),
transforms.Resize(scale),
transforms.CenterCrop(self.output_size),
transforms.HorizontalFlip(should_flip),
])
rgb = base_transform(rgb)
rgb = self.color_jitter(rgb)
rgb = rgb / 255.0
depth = base_transform(depth)
return (rgb, depth)
def train_transform(self, rgb, depth, rgb_near):
s = np.random.uniform(1.0, 1.5) # random scaling
depth_np = depth / s
angle = np.random.uniform(-5.0, 5.0) # random rotation degrees
do_flip = np.random.uniform(0.0, 1.0) < 0.5 # random horizontal flip
# perform 1st step of data augmentation
transform = transforms.Compose([
transforms.Resize(
250.0 / iheight
), # this is for computational efficiency, since rotation can be slow
transforms.Rotate(angle),
transforms.Resize(s),
transforms.CenterCrop(self.output_size),
transforms.HorizontalFlip(do_flip)
])
rgb_np = transform(rgb)
rgb_np = self.color_jitter(rgb_np) # random color jittering
rgb_np = np.asfarray(rgb_np, dtype='float') / 255
rgb_near_np = None
if rgb_near is not None:
rgb_near_np = transform(rgb_near)
rgb_near_np = np.asfarray(rgb_near_np, dtype='float') / 255
depth_np = transform(depth_np)
self.K = TransfromIntrinsics(self.K, (250.0 / iheight) * s,
self.output_size)
return rgb_np, depth_np, rgb_near_np
def train_transform(self, rgb, depth):
s = np.random.uniform(1.0, 1.5) # random scaling
depth_np = depth #/ s
angle = np.random.uniform(-5.0, 5.0) # random rotation degrees
do_flip = np.random.uniform(0.0, 1.0) < 0.5 # random horizontal flip
# perform 1st step of data augmentation
transform = transforms.Compose([
transforms.Resize(
240.0 / iheight
), # this is for computational efficiency, since rotation can be slow
#transforms.Rotate(angle),
#transforms.Resize(s),
transforms.CenterCrop(self.output_size),
transforms.HorizontalFlip(do_flip)
])
rgb_np = transform(rgb)
#rgb_np = self.color_jitter(rgb_np) # random color jittering
rgb_np = np.asfarray(rgb_np, dtype='float') / 255
depth_np = transform(depth_np)
depth_np = np.asfarray(depth_np, dtype='float')
if self.depth_16:
depth_np = depth_np / self.depth_16_max
else:
depth_np = (255 - depth_np) / 255
return rgb_np, depth_np
def train_transform(self, rgb, depth):
# for create fake underwater images
rgb = uw_style(rgb, depth)
rgb /= rgb.max() / 255
rgb = rgb.astype(np.uint8)
s = np.random.uniform(1.0, 1.5) # random scaling
depth_np = depth / s
angle = np.random.uniform(-5.0, 5.0) # random rotation degrees
do_flip = np.random.uniform(0.0, 1.0) < 0.5 # random horizontal flip
# perform 1st step of data augmentation
transform = transforms.Compose([
# transforms.Rotate(angle),
# transforms.Resize(s),
# transforms.CenterCrop(self.output_size),
transforms.HorizontalFlip(do_flip),
transforms.Resize(size=self.output_size)
])
rgb_np = transform(rgb)
rgb_np = self.color_jitter(rgb_np) # random color jittering
rgb_np = np.asfarray(rgb_np, dtype='float') / 255
depth_np = transform(depth_np)
return rgb_np, depth_np
def train_transform(self, im, gt, mask):
im = np.array(im).astype(np.float32)
im = cv2.resize(im, (512, 256), interpolation=cv2.INTER_AREA)
gt = cv2.resize(gt, (512, 256), interpolation=cv2.INTER_AREA)
mask = cv2.resize(mask, (512, 256), interpolation=cv2.INTER_AREA)
# h,w,c = im.shape
# th, tw = 256,512
# x1 = random.randint(0, w - tw)
# y1 = random.randint(0, h - th)
# img = im[y1:y1 + th, x1:x1 + tw, :]
# gt = gt[y1:y1 + th, x1:x1 + tw]
# mask = mask[y1:y1 + th, x1:x1 + tw]
s = np.random.uniform(1.0, 1.5) # random scaling
angle = np.random.uniform(-5.0, 5.0) # random rotation degrees
do_flip = np.random.uniform(0.0, 1.0) < 0.5 # random horizontal flip
color_jitter = my_transforms.ColorJitter(0.4, 0.4, 0.4)
transform = my_transforms.Compose([
my_transforms.Rotate(angle),
my_transforms.Resize(s),
my_transforms.CenterCrop(self.size),
my_transforms.HorizontalFlip(do_flip)
])
im_ = transform(im)
im_ = color_jitter(im_)
gt_ = transform(gt)
mask_ = transform(mask)
im_ = np.array(im_).astype(np.float32)
gt_ = np.array(gt_).astype(np.float32)
mask_ = np.array(mask_).astype(np.float32)
im_ /= 255.0
gt_ /= s
im_ = to_tensor(im_)
gt_ = to_tensor(gt_)
mask_ = to_tensor(mask_)
gt_ = gt_.unsqueeze(0)
mask_ = mask_.unsqueeze(0)
return im_, gt_, mask_
def train_transform(self, rgb, depth):
"""
[Reference]
https://github.com/fangchangma/sparse-to-dense.pytorch/blob/master/dataloaders/nyu_dataloader.py
Args:
rgb (np.array): RGB image (shape=[H,W,3])
depth (np.array): Depth image (shape=[H,W])
Returns:
torch.Tensor: Tranformed RGB image
torch.Tensor: Transformed Depth image
np.array: Transformed RGB image without color jitter (for 2D mesh creation)
"""
# Parameters for each augmentation
s = np.random.uniform(1.0, 1.5) # random scaling
depth_np = depth / s
angle = np.random.uniform(-5.0, 5.0) # random rotation degrees
do_flip = np.random.uniform(0.0, 1.0) < 0.5 # random horizontal flip
# Perform 1st step of data augmentation
transform = transforms.Compose([
transforms.Resize(250.0 / RAW_HEIGHT),
transforms.Rotate(angle),
transforms.Resize(s),
transforms.CenterCrop(self.img_size),
transforms.HorizontalFlip(do_flip)
])
# Apply this transform to rgb/depth
rgb_np_orig = transform(rgb)
rgb_np_for_edge = np.asfarray(
rgb_np_orig) # Used for canny edge detection
rgb_np = color_jitter(rgb_np_orig) # random color jittering
rgb_np = np.asfarray(rgb_np) / 255
depth_np = transform(depth_np)
return rgb_np, depth_np, rgb_np_for_edge
def train_transform(self, rgb, depth):
s = np.random.uniform(1.0, 1.5) # random scaling
depth_np = depth / (s * self.depth_divider)
angle = np.random.uniform(-5.0, 5.0) # random rotation degrees
do_flip = np.random.uniform(0.0, 1.0) < 0.5 # random horizontal flip
# perform 1st step of data augmentation
transform = transforms.Compose([
transforms.Crop(130, 10, 240, 1200),
transforms.Rotate(angle),
transforms.Resize(s),
transforms.CenterCrop(self.output_size),
transforms.HorizontalFlip(do_flip)
])
rgb_np = transform(rgb)
rgb_np = self.color_jitter(rgb_np) # random color jittering
rgb_np = np.asfarray(rgb_np, dtype='float') / 255
# Scipy affine_transform produced RuntimeError when the depth map was
# given as a 'numpy.ndarray'
depth_np = np.asfarray(depth_np, dtype='float32')
depth_np = transform(depth_np)
return rgb_np, depth_np
def train_transform(self, rgb, depth):
s = np.random.uniform(1.0, 1.5) # random scaling
depth_np = depth / s
angle = np.random.uniform(-5.0, 5.0) # random rotation degrees
do_flip = np.random.uniform(0.0, 1.0) < 0.5 # random horizontal flip
# perform 1st step of data augmentation
transform = transforms.Compose([
#Why not crop like in KITTI? Also, if resizing done, why not reflect this in depth_np as well?
transforms.Resize(
250.0 / iheight
), # this is for computational efficiency, since rotation can be slow
transforms.Rotate(angle),
transforms.Resize(s),
transforms.CenterCrop(self.output_size),
transforms.HorizontalFlip(do_flip)
])
rgb_np = transform(rgb)
rgb_np = self.color_jitter(rgb_np) # random color jittering
rgb_np = np.asfarray(rgb_np, dtype='float') / 255
depth_np = transform(depth_np)
return rgb_np, depth_np
def train_transform(self, rgb, depth):
s = np.random.uniform(1.0, 1.5) # random scaling
depth_np = depth / s
angle = np.random.uniform(-5.0, 5.0) # random rotation degrees
do_flip = np.random.uniform(0.0, 1.0) < 0.5 # random horizontal flip
iheight = rgb.shape[0]
# perform 1st step of data augmentation
transform = transforms.Compose([
transforms.Resize(250.0 / iheight), # this is for computational efficiency, since rotation can be slow
transforms.Rotate(angle),
transforms.Resize(s),
transforms.CenterCrop((228, 304)),
transforms.HorizontalFlip(do_flip),
transforms.Resize(self.output_size),
])
rgb_np = transform(rgb)
rgb_np = self.color_jitter(rgb_np) # random color jittering
rgb_np = np.asfarray(rgb_np, dtype='float') / 255
if depth_np.ndim != 2:
print("Wrong Depth ",depth_np)
depth_np = transform(depth_np)
return rgb_np, depth_np
def train_transform(self, rgb, depth):
s = np.random.uniform(1.0, 1.5) # random scaling
# s = 1.5
depth_np = depth / s
angle = np.random.uniform(-5.0, 5.0) # random rotation degrees
do_flip = np.random.uniform(0.0, 1.0) < 0.5 # random horizontal flip
# perform 1st step of data augmentation
transform = transforms.Compose([
transforms.Resize(
250.0 / iheight
), # this is for computational efficiency, since rotation can be slow
transforms.Rotate(angle),
transforms.Resize(
s), # TODO (Katie): figure out how to resize properly
transforms.RandomCrop(self.output_size),
transforms.HorizontalFlip(do_flip)
])
rgb_np = transform(rgb)
rgb_np = self.color_jitter(rgb_np) # random color jittering
rgb_np = np.asfarray(rgb_np, dtype='float') / 255
depth_np = transform(depth_np)
return rgb_np, depth_np
def _train_transform(self, rgb, sparse_depth, depth_gt):
s = np.random.uniform(1.0, 1.5) # random scaling
depth_gt = depth_gt / s
# TODO critical why is the input not scaled in original implementation?
sparse_depth = sparse_depth / s
# TODO adapt and refactor
angle = np.random.uniform(-5.0, 5.0) # random rotation degrees
do_flip = np.random.uniform(0.0, 1.0) < 0.5 # random horizontal flip
# perform 1st step of data augmentation
# TODO critical adjust sizes
transform = transforms.Compose([
transforms.Crop(*self._road_crop),
transforms.Rotate(angle),
transforms.Resize(s),
transforms.CenterCrop(self.output_size),
transforms.HorizontalFlip(do_flip)
])
rgb = transform(rgb)
sparse_depth = transform(sparse_depth)
# TODO needed?
# Scipy affine_transform produced RuntimeError when the depth map was
# given as a 'numpy.ndarray'
depth_gt = np.asfarray(depth_gt, dtype='float32')
depth_gt = transform(depth_gt)
rgb = self._color_jitter(rgb) # random color jittering
# convert color [0,255] -> [0.0, 1.0] floats
rgb = np.asfarray(rgb, dtype='float') / 255
return rgb, sparse_depth, depth_gt
def train_transform(self, attrib_list):
iheight = attrib_list['gt_depth'].shape[0]
iwidth = attrib_list['gt_depth'].shape[1]
s = np.random.uniform(1.0, 1.5) # random scaling
angle = np.random.uniform(-15.0, 15.0) # random rotation degrees
hdo_flip = np.random.uniform(0.0, 1.0) < 0.5 # random horizontal flip
vdo_flip = np.random.uniform(0.0, 1.0) < 0.5 # random vertical flip
# perform 1st step of data augmentation
transform = transforms.Compose([
transforms.Resize(
270.0 / iheight
), # this is for computational efficiency, since rotation can be slow
transforms.Rotate(angle),
transforms.Resize(s),
transforms.CenterCrop(self.output_size),
transforms.HorizontalFlip(hdo_flip),
transforms.VerticalFlip(vdo_flip)
])
attrib_np = dict()
if self.depth_divider == 0:
if 'fd' in attrib_list:
minmax_image = transform(attrib_list['fd'])
max_depth = max(minmax_image.max(), 1.0)
if 'kor' in attrib_list:
minmax_image = transform(attrib_list['kor'])
max_depth = max(minmax_image.max(), 1.0)
else:
max_depth = 50
scale = 10.0 / max_depth # 10 is arbitrary. the network only converge in a especific range
else:
scale = 1.0 / self.depth_divider
attrib_np['scale'] = 1.0 / scale
for key, value in attrib_list.items():
attrib_np[key] = transform(value)
if key in Modality.need_divider: #['gt_depth','fd','kor','kde','kgt','dor','dde', 'd3dwde','d3dwor','dvor','dvde','dvgt']:
attrib_np[key] = scale * attrib_np[
key] #(attrib_np[key] - min_depth+0.01) / (max_depth - min_depth) #/
elif key in Modality.image_size_weight_names: #['d2dwor', 'd2dwde', 'd2dwgt']:
attrib_np[key] = attrib_np[key] / (
iwidth * 1.5) # 1.5 about sqrt(2)- square's diagonal
if 'rgb' in attrib_np:
attrib_np['rgb'] = self.color_jitter(
attrib_np['rgb']) # random color jittering
attrib_np['rgb'] = (np.asfarray(attrib_np['rgb'], dtype='float') /
255).transpose(
(2, 0,
1)) #all channels need to have C x H x W
if 'grey' in attrib_np:
attrib_np['grey'] = np.expand_dims(
np.asfarray(attrib_np['grey'], dtype='float') / 255, axis=0)
return attrib_np
def train_transform(rgb, sparse, target, position, args):
# s = np.random.uniform(1.0, 1.5) # random scaling
# angle = np.random.uniform(-5.0, 5.0) # random rotation degrees
oheight = args.val_h
owidth = args.val_w
do_flip = np.random.uniform(0.0, 1.0) < 0.5 # random horizontal flip
transforms_list = [
# transforms.Rotate(angle),
# transforms.Resize(s),
transforms.BottomCrop((oheight, owidth)),
transforms.HorizontalFlip(do_flip)
]
# if small_training == True:
# transforms_list.append(transforms.RandomCrop((rheight, rwidth)))
transform_geometric = transforms.Compose(transforms_list)
if sparse is not None:
sparse = transform_geometric(sparse)
target = transform_geometric(target)
if rgb is not None:
brightness = np.random.uniform(max(0, 1 - args.jitter),
1 + args.jitter)
contrast = np.random.uniform(max(0, 1 - args.jitter), 1 + args.jitter)
saturation = np.random.uniform(max(0, 1 - args.jitter),
1 + args.jitter)
transform_rgb = transforms.Compose([
transforms.ColorJitter(brightness, contrast, saturation, 0),
transform_geometric
])
rgb = transform_rgb(rgb)
# sparse = drop_depth_measurements(sparse, 0.9)
if position is not None:
bottom_crop_only = transforms.Compose(
[transforms.BottomCrop((oheight, owidth))])
position = bottom_crop_only(position)
# random crop
#if small_training == True:
if args.not_random_crop == False:
h = oheight
w = owidth
rheight = args.random_crop_height
rwidth = args.random_crop_width
# randomlize
i = np.random.randint(0, h - rheight + 1)
j = np.random.randint(0, w - rwidth + 1)
if rgb is not None:
if rgb.ndim == 3:
rgb = rgb[i:i + rheight, j:j + rwidth, :]
elif rgb.ndim == 2:
rgb = rgb[i:i + rheight, j:j + rwidth]
if sparse is not None:
if sparse.ndim == 3:
sparse = sparse[i:i + rheight, j:j + rwidth, :]
elif sparse.ndim == 2:
sparse = sparse[i:i + rheight, j:j + rwidth]
if target is not None:
if target.ndim == 3:
target = target[i:i + rheight, j:j + rwidth, :]
elif target.ndim == 2:
target = target[i:i + rheight, j:j + rwidth]
if position is not None:
if position.ndim == 3:
position = position[i:i + rheight, j:j + rwidth, :]
elif position.ndim == 2:
position = position[i:i + rheight, j:j + rwidth]
return rgb, sparse, target, position