Exemple #1
0
def _2d_data_aug_fn(rgb_image, depth_list, ground_truth2d):
    """Data augmentation function."""
    annos2d = cPickle.loads(ground_truth2d)
    annos2d = list(annos2d)

    depth_image = sio.loadmat(depth_list)['depthim_incolor']
    depth_image = depth_image / 20.0

    ## 2d data augmentation
    # random transfrom
    M_rotate = tl.prepro.affine_rotation_matrix(angle=(-30, 30))  # original paper: -40~40
    M_zoom = tl.prepro.affine_zoom_matrix(zoom_range=(0.5, 0.8))  # original paper: 0.5~1.1
    M_combined = M_rotate.dot(M_zoom)
    transform_matrix = tl.prepro.transform_matrix_offset_center(M_combined, x=rgb_image.shape[0], y=rgb_image.shape[1])
    rgb_image = tl.prepro.affine_transform_cv2(rgb_image, transform_matrix)
    depth_image = tl.prepro.affine_transform_cv2(depth_image, transform_matrix, border_mode='replicate')
    annos2d = tl.prepro.affine_transform_keypoints(annos2d, transform_matrix)
    # random crop and flip
    rgb_image, annos2d, depth_image = tl.prepro.keypoint_random_flip(rgb_image, annos2d, depth_image, prob=0.5)    
    rgb_image, annos2d, depth_image = tl.prepro.keypoint_resize_random_crop(rgb_image, annos2d, depth_image, size=(hin, win)) # hao add
    # concat augmented rgb and depth
    depth_image = np.expand_dims(depth_image, axis=2)
    input_2d = np.concatenate((rgb_image, depth_image), axis=2)

    # generate 2d result maps including keypoints heatmap, pafs
    height, width, _ = input_2d.shape
    heatmap = get_heatmap(annos2d, height, width)
    vectormap = get_vectormap(annos2d, height, width)
    result2dmap = np.concatenate((heatmap, vectormap), axis=2)

    input_2d = np.array(input_2d, dtype=np.float32)
    result2dmap = np.array(result2dmap, dtype=np.float32)

    return input_2d, result2dmap
Exemple #2
0
def _data_aug_fn(image, ground_truth):
    """Data augmentation function."""
    ground_truth = cPickle.loads(ground_truth)
    ground_truth = list(ground_truth)

    annos = ground_truth[0]
    mask = ground_truth[1]
    h_mask, w_mask, _ = np.shape(image)
    # mask
    mask_miss = np.ones((h_mask, w_mask), dtype=np.uint8)

    for seg in mask:
        bin_mask = maskUtils.decode(seg)
        bin_mask = np.logical_not(bin_mask)
        mask_miss = np.bitwise_and(mask_miss, bin_mask)

    ## image data augmentation
    # randomly resize height and width independently, scale is changed
    image, annos, mask_miss = keypoint_random_resize(image, annos, mask_miss, zoom_range=(0.8, 1.2))
    # random rotate
    image, annos, mask_miss = keypoint_random_rotate(image, annos, mask_miss, rg=15.0)
    # random left-right flipping
    image, annos, mask_miss = keypoint_random_flip(image, annos, mask_miss, prob=0.5)
    # random resize height and width together
    image, annos, mask_miss = keypoint_random_resize_shortestedge(
        image, annos, mask_miss, min_size=(hin, win), zoom_range=(0.95, 1.6))
    # random crop
    image, annos, mask_miss = keypoint_random_crop(image, annos, mask_miss, size=(hin, win))  # with padding

    # generate result maps including keypoints heatmap, pafs and mask
    h, w, _ = np.shape(image)
    height, width, _ = np.shape(image)
    heatmap = get_heatmap(annos, height, width)
    vectormap = get_vectormap(annos, height, width)
    resultmap = np.concatenate((heatmap, vectormap), axis=2)

    image = np.array(image, dtype=np.float32)

    img_mask = mask_miss.reshape(hin, win, 1)
    image = image * np.repeat(img_mask, 3, 2)

    resultmap = np.array(resultmap, dtype=np.float32)
    mask_miss = cv2.resize(mask_miss, (hout, wout), interpolation=cv2.INTER_AREA)
    mask_miss = np.array(mask_miss, dtype=np.float32)
    return image, resultmap, mask_miss
Exemple #3
0
def _mock_data_aug_fn(image, ground_truth):
    """Data augmentation function."""
    ground_truth = cPickle.loads(ground_truth)
    ground_truth = list(ground_truth)

    annos = ground_truth[0]
    mask = ground_truth[1]
    h_mask, w_mask, _ = np.shape(image)

    # mask
    mask_miss = np.ones((h_mask, w_mask), dtype=np.uint8)

    for seg in mask:
        bin_mask = maskUtils.decode(seg)
        bin_mask = np.logical_not(bin_mask)
        mask_miss = np.bitwise_and(mask_miss, bin_mask)

    # random crop
    #TODO only working with quadradic dimmentions
    image, annos, mask_miss = tl.prepro.keypoint_resize_random_crop(
        image, annos, mask_miss,
        size=(config.MODEL.hin, config.MODEL.win))  # hao add

    # generate result maps including keypoints heatmap, pafs and mask
    height, width, _ = np.shape(image)

    heatmap = get_heatmap(annos, height, width)
    vectormap = get_vectormap(annos, height, width)
    resultmap = np.concatenate((heatmap, vectormap), axis=2)

    image = np.array(image, dtype=np.float32)

    #TODO image has not always the right size if  256 * 384 is requested
    print("image1 ", height, width, _)
    print("mask1 ", mask_miss.shape)

    # mask image in all 3 channels
    img_mask = mask_miss.reshape(height, width, 1)
    image = image * np.repeat(img_mask, 3, 2)

    resultmap = np.array(resultmap, dtype=np.float32)
    mask_miss = cv2.resize(mask_miss, (config.MODEL.hout, config.MODEL.wout),
                           interpolation=cv2.INTER_AREA)
    mask_miss = np.array(mask_miss, dtype=np.float32)
    return image, resultmap, mask_miss
Exemple #4
0
def _data_aug_fn(image, ground_truth):
    """Data augmentation function."""
    ground_truth = cPickle.loads(ground_truth)
    ground_truth = list(ground_truth)

    annos = ground_truth[0]
    mask = ground_truth[1]
    h_mask, w_mask, _ = np.shape(image)
    # mask
    mask_miss = np.ones((h_mask, w_mask), dtype=np.uint8)

    for seg in mask:
        bin_mask = maskUtils.decode(seg)
        bin_mask = np.logical_not(bin_mask)
        mask_miss = np.bitwise_and(mask_miss, bin_mask)

    ## image data augmentation
    # # randomly resize height and width independently, scale is changed
    # image, annos, mask_miss = keypoint_random_resize(image, annos, mask_miss, zoom_range=(0.8, 1.2))# removed hao
    # # random rotate
    # image, annos, mask_miss = keypoint_random_rotate(image, annos, mask_miss, rg=15.0)# removed hao
    # # random left-right flipping
    # image, annos, mask_miss = keypoint_random_flip(image, annos, mask_miss, prob=0.5)# removed hao
    M_rotate = tl.prepro.affine_rotation_matrix(angle=(-30, 30))  # -40~40
    M_flip = tl.prepro.affine_horizontal_flip_matrix(prob=0.5)
    M_zoom = tl.prepro.affine_zoom_matrix(zoom_range=(0.5, 0.8))  # 0.5~1.1
    # M_shear = tl.prepro.affine_shear_matrix(x_shear=(-0.1, 0.1), y_shear=(-0.1, 0.1))
    M_combined = M_rotate.dot(M_flip).dot(M_zoom)  #.dot(M_shear)
    # M_combined = tl.prepro.affine_zoom_matrix(zoom_range=0.9) # for debug
    h, w, _ = image.shape
    transform_matrix = tl.prepro.transform_matrix_offset_center(M_combined,
                                                                x=w,
                                                                y=h)
    image = tl.prepro.affine_transform_cv2(image, transform_matrix)
    annos = tl.prepro.affine_transform_keypoints(annos, transform_matrix)

    # random resize height and width together
    # image, annos, mask_miss = keypoint_random_resize_shortestedge(
    #     image, annos, mask_miss, min_size=(hin, win), zoom_range=(0.95, 1.6)) # removed hao
    # random crop
    # image, annos, mask_miss = keypoint_random_crop(image, annos, mask_miss, size=(hin, win))  # with padding # removed hao
    image, annos, mask_miss = tl.prepro.keypoint_resize_random_crop(image,
                                                                    annos,
                                                                    mask_miss,
                                                                    size=(hin,
                                                                          win))

    # generate result maps including keypoints heatmap, pafs and mask
    h, w, _ = np.shape(image)
    height, width, _ = np.shape(image)
    heatmap = get_heatmap(annos, height, width)
    vectormap = get_vectormap(annos, height, width)
    resultmap = np.concatenate((heatmap, vectormap), axis=2)

    image = np.array(image, dtype=np.float32)

    img_mask = mask_miss.reshape(hin, win, 1)
    image = image * np.repeat(img_mask, 3, 2)

    resultmap = np.array(resultmap, dtype=np.float32)
    mask_miss = cv2.resize(mask_miss, (hout, wout),
                           interpolation=cv2.INTER_AREA)
    mask_miss = np.array(mask_miss, dtype=np.float32)
    return image, resultmap, mask_miss
Exemple #5
0
def _data_aug_fn(image, ground_truth):
    """Data augmentation function."""
    ground_truth = cPickle.loads(ground_truth)
    ground_truth = list(ground_truth)

    annos = ground_truth[0]
    mask = ground_truth[1]
    h_mask, w_mask, _ = np.shape(image)

    # mask
    mask_miss = np.ones((h_mask, w_mask), dtype=np.uint8)

    #TODO only working with quadradic dimmentions
    #print("image0 ",h_mask, w_mask, _)
    #print("mask0 ",mask_miss.shape)

    for seg in mask:
        bin_mask = maskUtils.decode(seg)
        bin_mask = np.logical_not(bin_mask)
        mask_miss = np.bitwise_and(mask_miss, bin_mask)

    ## image data augmentation
    # # randomly resize height and width independently, scale is changed
    # image, annos, mask_miss = tl.prepro.keypoint_random_resize(image, annos, mask_miss, zoom_range=(0.8, 1.2))# removed hao
    # # random rotate
    # image, annos, mask_miss = tl.prepro.keypoint_random_rotate(image, annos, mask_miss, rg=15.0)# removed hao
    # # random left-right flipping
    # image, annos, mask_miss = tl.prepro.keypoint_random_flip(image, annos, mask_miss, prob=0.5)# removed hao

    M_rotate = tl.prepro.affine_rotation_matrix(
        angle=(-180, 180))  # original paper: -40~40
    # M_flip = tl.prepro.affine_horizontal_flip_matrix(prob=0.5) # hao removed: bug, keypoints will have error
    M_zoom = tl.prepro.affine_zoom_matrix(
        zoom_range=(0.5, 1.1))  # original paper: 0.5~1.1
    # M_shear = tl.prepro.affine_shear_matrix(x_shear=(-0.1, 0.1), y_shear=(-0.1, 0.1))
    M_combined = M_rotate.dot(M_zoom)
    # M_combined = M_rotate.dot(M_flip).dot(M_zoom)#.dot(M_shear)
    # M_combined = tl.prepro.affine_zoom_matrix(zoom_range=0.9) # for debug
    h, w, _ = image.shape

    transform_matrix = tl.prepro.transform_matrix_offset_center(M_combined,
                                                                x=w,
                                                                y=h)
    image = tl.prepro.affine_transform_cv2(image, transform_matrix)
    mask_miss = tl.prepro.affine_transform_cv2(mask_miss,
                                               transform_matrix,
                                               border_mode='replicate')
    annos = tl.prepro.affine_transform_keypoints(annos, transform_matrix)

    # random resize height and width together
    # image, annos, mask_miss = tl.prepro.keypoint_random_resize_shortestedge(
    #     image, annos, mask_miss, min_size=(hin, win), zoom_range=(0.95, 1.6)) # removed hao
    image, annos, mask_miss = tl.prepro.keypoint_random_flip(image,
                                                             annos,
                                                             mask_miss,
                                                             prob=0.5)

    # random crop
    #TODO only working with quadradic dimmentions
    image, annos, mask_miss = tl.prepro.keypoint_resize_random_crop(
        image, annos, mask_miss,
        size=(config.MODEL.hin, config.MODEL.win))  # hao add

    # generate result maps including keypoints heatmap, pafs and mask
    height, width, _ = np.shape(image)

    heatmap = get_heatmap(annos, height, width)
    vectormap = get_vectormap(annos, height, width)
    resultmap = np.concatenate((heatmap, vectormap), axis=2)

    image = np.array(image, dtype=np.float32)

    #TODO image has not always the right size if  256 * 384 is requested
    #print("image1 ",height, width, _)
    #print("mask1 ",mask_miss.shape)

    # mask image in all 3 channels
    img_mask = mask_miss.reshape(height, width, 1)
    image = image * np.repeat(img_mask, 3, 2)

    resultmap = np.array(resultmap, dtype=np.float32)
    mask_miss = cv2.resize(mask_miss, (config.MODEL.hout, config.MODEL.wout),
                           interpolation=cv2.INTER_AREA)
    mask_miss = np.array(mask_miss, dtype=np.float32)
    return image, resultmap, mask_miss