def _2d_data_aug_fn(rgb_image, depth_list, ground_truth2d):
"""Data augmentation function."""
annos2d = cPickle.loads(ground_truth2d)
annos2d = list(annos2d)
depth_image = sio.loadmat(depth_list)['depthim_incolor']
depth_image = depth_image / 20.0
## 2d data augmentation
# random transfrom
M_rotate = tl.prepro.affine_rotation_matrix(angle=(-30, 30)) # original paper: -40~40
M_zoom = tl.prepro.affine_zoom_matrix(zoom_range=(0.5, 0.8)) # original paper: 0.5~1.1
M_combined = M_rotate.dot(M_zoom)
transform_matrix = tl.prepro.transform_matrix_offset_center(M_combined, x=rgb_image.shape[0], y=rgb_image.shape[1])
rgb_image = tl.prepro.affine_transform_cv2(rgb_image, transform_matrix)
depth_image = tl.prepro.affine_transform_cv2(depth_image, transform_matrix, border_mode='replicate')
annos2d = tl.prepro.affine_transform_keypoints(annos2d, transform_matrix)
# random crop and flip
rgb_image, annos2d, depth_image = tl.prepro.keypoint_random_flip(rgb_image, annos2d, depth_image, prob=0.5)
rgb_image, annos2d, depth_image = tl.prepro.keypoint_resize_random_crop(rgb_image, annos2d, depth_image, size=(hin, win)) # hao add
# concat augmented rgb and depth
depth_image = np.expand_dims(depth_image, axis=2)
input_2d = np.concatenate((rgb_image, depth_image), axis=2)
# generate 2d result maps including keypoints heatmap, pafs
height, width, _ = input_2d.shape
heatmap = get_heatmap(annos2d, height, width)
vectormap = get_vectormap(annos2d, height, width)
result2dmap = np.concatenate((heatmap, vectormap), axis=2)
input_2d = np.array(input_2d, dtype=np.float32)
result2dmap = np.array(result2dmap, dtype=np.float32)
return input_2d, result2dmap
def _data_aug_fn(image, ground_truth):
"""Data augmentation function."""
ground_truth = cPickle.loads(ground_truth)
ground_truth = list(ground_truth)
annos = ground_truth[0]
mask = ground_truth[1]
h_mask, w_mask, _ = np.shape(image)
# mask
mask_miss = np.ones((h_mask, w_mask), dtype=np.uint8)
for seg in mask:
bin_mask = maskUtils.decode(seg)
bin_mask = np.logical_not(bin_mask)
mask_miss = np.bitwise_and(mask_miss, bin_mask)
## image data augmentation
# randomly resize height and width independently, scale is changed
image, annos, mask_miss = keypoint_random_resize(image, annos, mask_miss, zoom_range=(0.8, 1.2))
# random rotate
image, annos, mask_miss = keypoint_random_rotate(image, annos, mask_miss, rg=15.0)
# random left-right flipping
image, annos, mask_miss = keypoint_random_flip(image, annos, mask_miss, prob=0.5)
# random resize height and width together
image, annos, mask_miss = keypoint_random_resize_shortestedge(
image, annos, mask_miss, min_size=(hin, win), zoom_range=(0.95, 1.6))
# random crop
image, annos, mask_miss = keypoint_random_crop(image, annos, mask_miss, size=(hin, win)) # with padding
# generate result maps including keypoints heatmap, pafs and mask
h, w, _ = np.shape(image)
height, width, _ = np.shape(image)
heatmap = get_heatmap(annos, height, width)
vectormap = get_vectormap(annos, height, width)
resultmap = np.concatenate((heatmap, vectormap), axis=2)
image = np.array(image, dtype=np.float32)
img_mask = mask_miss.reshape(hin, win, 1)
image = image * np.repeat(img_mask, 3, 2)
resultmap = np.array(resultmap, dtype=np.float32)
mask_miss = cv2.resize(mask_miss, (hout, wout), interpolation=cv2.INTER_AREA)
mask_miss = np.array(mask_miss, dtype=np.float32)
return image, resultmap, mask_miss
def _mock_data_aug_fn(image, ground_truth):
"""Data augmentation function."""
ground_truth = cPickle.loads(ground_truth)
ground_truth = list(ground_truth)
annos = ground_truth[0]
mask = ground_truth[1]
h_mask, w_mask, _ = np.shape(image)
# mask
mask_miss = np.ones((h_mask, w_mask), dtype=np.uint8)
for seg in mask:
bin_mask = maskUtils.decode(seg)
bin_mask = np.logical_not(bin_mask)
mask_miss = np.bitwise_and(mask_miss, bin_mask)
# random crop
#TODO only working with quadradic dimmentions
image, annos, mask_miss = tl.prepro.keypoint_resize_random_crop(
image, annos, mask_miss,
size=(config.MODEL.hin, config.MODEL.win)) # hao add
# generate result maps including keypoints heatmap, pafs and mask
height, width, _ = np.shape(image)
heatmap = get_heatmap(annos, height, width)
vectormap = get_vectormap(annos, height, width)
resultmap = np.concatenate((heatmap, vectormap), axis=2)
image = np.array(image, dtype=np.float32)
#TODO image has not always the right size if 256 * 384 is requested
print("image1 ", height, width, _)
print("mask1 ", mask_miss.shape)
# mask image in all 3 channels
img_mask = mask_miss.reshape(height, width, 1)
image = image * np.repeat(img_mask, 3, 2)
resultmap = np.array(resultmap, dtype=np.float32)
mask_miss = cv2.resize(mask_miss, (config.MODEL.hout, config.MODEL.wout),
interpolation=cv2.INTER_AREA)
mask_miss = np.array(mask_miss, dtype=np.float32)
return image, resultmap, mask_miss
def _data_aug_fn(image, ground_truth):
"""Data augmentation function."""
ground_truth = cPickle.loads(ground_truth)
ground_truth = list(ground_truth)
annos = ground_truth[0]
mask = ground_truth[1]
h_mask, w_mask, _ = np.shape(image)
# mask
mask_miss = np.ones((h_mask, w_mask), dtype=np.uint8)
for seg in mask:
bin_mask = maskUtils.decode(seg)
bin_mask = np.logical_not(bin_mask)
mask_miss = np.bitwise_and(mask_miss, bin_mask)
## image data augmentation
# # randomly resize height and width independently, scale is changed
# image, annos, mask_miss = keypoint_random_resize(image, annos, mask_miss, zoom_range=(0.8, 1.2))# removed hao
# # random rotate
# image, annos, mask_miss = keypoint_random_rotate(image, annos, mask_miss, rg=15.0)# removed hao
# # random left-right flipping
# image, annos, mask_miss = keypoint_random_flip(image, annos, mask_miss, prob=0.5)# removed hao
M_rotate = tl.prepro.affine_rotation_matrix(angle=(-30, 30)) # -40~40
M_flip = tl.prepro.affine_horizontal_flip_matrix(prob=0.5)
M_zoom = tl.prepro.affine_zoom_matrix(zoom_range=(0.5, 0.8)) # 0.5~1.1
# M_shear = tl.prepro.affine_shear_matrix(x_shear=(-0.1, 0.1), y_shear=(-0.1, 0.1))
M_combined = M_rotate.dot(M_flip).dot(M_zoom) #.dot(M_shear)
# M_combined = tl.prepro.affine_zoom_matrix(zoom_range=0.9) # for debug
h, w, _ = image.shape
transform_matrix = tl.prepro.transform_matrix_offset_center(M_combined,
x=w,
y=h)
image = tl.prepro.affine_transform_cv2(image, transform_matrix)
annos = tl.prepro.affine_transform_keypoints(annos, transform_matrix)
# random resize height and width together
# image, annos, mask_miss = keypoint_random_resize_shortestedge(
# image, annos, mask_miss, min_size=(hin, win), zoom_range=(0.95, 1.6)) # removed hao
# random crop
# image, annos, mask_miss = keypoint_random_crop(image, annos, mask_miss, size=(hin, win)) # with padding # removed hao
image, annos, mask_miss = tl.prepro.keypoint_resize_random_crop(image,
annos,
mask_miss,
size=(hin,
win))
# generate result maps including keypoints heatmap, pafs and mask
h, w, _ = np.shape(image)
height, width, _ = np.shape(image)
heatmap = get_heatmap(annos, height, width)
vectormap = get_vectormap(annos, height, width)
resultmap = np.concatenate((heatmap, vectormap), axis=2)
image = np.array(image, dtype=np.float32)
img_mask = mask_miss.reshape(hin, win, 1)
image = image * np.repeat(img_mask, 3, 2)
resultmap = np.array(resultmap, dtype=np.float32)
mask_miss = cv2.resize(mask_miss, (hout, wout),
interpolation=cv2.INTER_AREA)
mask_miss = np.array(mask_miss, dtype=np.float32)
return image, resultmap, mask_miss
def _data_aug_fn(image, ground_truth):
"""Data augmentation function."""
ground_truth = cPickle.loads(ground_truth)
ground_truth = list(ground_truth)
annos = ground_truth[0]
mask = ground_truth[1]
h_mask, w_mask, _ = np.shape(image)
# mask
mask_miss = np.ones((h_mask, w_mask), dtype=np.uint8)
#TODO only working with quadradic dimmentions
#print("image0 ",h_mask, w_mask, _)
#print("mask0 ",mask_miss.shape)
for seg in mask:
bin_mask = maskUtils.decode(seg)
bin_mask = np.logical_not(bin_mask)
mask_miss = np.bitwise_and(mask_miss, bin_mask)
## image data augmentation
# # randomly resize height and width independently, scale is changed
# image, annos, mask_miss = tl.prepro.keypoint_random_resize(image, annos, mask_miss, zoom_range=(0.8, 1.2))# removed hao
# # random rotate
# image, annos, mask_miss = tl.prepro.keypoint_random_rotate(image, annos, mask_miss, rg=15.0)# removed hao
# # random left-right flipping
# image, annos, mask_miss = tl.prepro.keypoint_random_flip(image, annos, mask_miss, prob=0.5)# removed hao
M_rotate = tl.prepro.affine_rotation_matrix(
angle=(-180, 180)) # original paper: -40~40
# M_flip = tl.prepro.affine_horizontal_flip_matrix(prob=0.5) # hao removed: bug, keypoints will have error
M_zoom = tl.prepro.affine_zoom_matrix(
zoom_range=(0.5, 1.1)) # original paper: 0.5~1.1
# M_shear = tl.prepro.affine_shear_matrix(x_shear=(-0.1, 0.1), y_shear=(-0.1, 0.1))
M_combined = M_rotate.dot(M_zoom)
# M_combined = M_rotate.dot(M_flip).dot(M_zoom)#.dot(M_shear)
# M_combined = tl.prepro.affine_zoom_matrix(zoom_range=0.9) # for debug
h, w, _ = image.shape
transform_matrix = tl.prepro.transform_matrix_offset_center(M_combined,
x=w,
y=h)
image = tl.prepro.affine_transform_cv2(image, transform_matrix)
mask_miss = tl.prepro.affine_transform_cv2(mask_miss,
transform_matrix,
border_mode='replicate')
annos = tl.prepro.affine_transform_keypoints(annos, transform_matrix)
# random resize height and width together
# image, annos, mask_miss = tl.prepro.keypoint_random_resize_shortestedge(
# image, annos, mask_miss, min_size=(hin, win), zoom_range=(0.95, 1.6)) # removed hao
image, annos, mask_miss = tl.prepro.keypoint_random_flip(image,
annos,
mask_miss,
prob=0.5)
# random crop
#TODO only working with quadradic dimmentions
image, annos, mask_miss = tl.prepro.keypoint_resize_random_crop(
image, annos, mask_miss,
size=(config.MODEL.hin, config.MODEL.win)) # hao add
# generate result maps including keypoints heatmap, pafs and mask
height, width, _ = np.shape(image)
heatmap = get_heatmap(annos, height, width)
vectormap = get_vectormap(annos, height, width)
resultmap = np.concatenate((heatmap, vectormap), axis=2)
image = np.array(image, dtype=np.float32)
#TODO image has not always the right size if 256 * 384 is requested
#print("image1 ",height, width, _)
#print("mask1 ",mask_miss.shape)
# mask image in all 3 channels
img_mask = mask_miss.reshape(height, width, 1)
image = image * np.repeat(img_mask, 3, 2)
resultmap = np.array(resultmap, dtype=np.float32)
mask_miss = cv2.resize(mask_miss, (config.MODEL.hout, config.MODEL.wout),
interpolation=cv2.INTER_AREA)
mask_miss = np.array(mask_miss, dtype=np.float32)
return image, resultmap, mask_miss