def __getitem__(self, index):
if self.is_train:
a = self.anno[self.train[index]]
else:
a = self.anno[self.valid[index]]
img_path = os.path.join(self.img_folder, a['img_paths'])
pts = torch.Tensor(a['joint_self'])
# pts[:, 0:2] -= 1 # Convert pts to zero based
pts = pts[:, 0:2]
# c = torch.Tensor(a['objpos']) - 1
c = torch.Tensor(a['objpos'])
# print c
s = torch.Tensor([a['scale_provided']])
# print s
# exit()
if a['dataset'] == 'MPII':
c[1] = c[1] + 15 * s[0]
s = s * 1.25
normalizer = a['normalizer'] * 0.6
elif a['dataset'] == 'LEEDS':
print 'using lsp data'
s = s * 1.4375
normalizer = torch.dist(pts[2, :], pts[13, :])
else:
print 'no such dataset {}'.format(a['dataset'])
# For single-person pose estimation with a centered/scaled figure
img = imutils.load_image(img_path)
# print img.size()
# exit()
# img = scipy.misc.imread(img_path, mode='RGB') # CxHxW
# img = torch.from_numpy(img)
r = 0
if self.is_train:
s = s * (2**(sample_from_bounded_gaussian(self.scale_factor)))
r = sample_from_bounded_gaussian(self.rot_factor)
if np.random.uniform(0, 1, 1) <= 0.6:
r = 0
# Flip
if np.random.random() <= 0.5:
img = torch.from_numpy(HumanAug.fliplr(img.numpy())).float()
pts = HumanAug.shufflelr(pts,
width=img.size(2),
dataset='mpii')
c[0] = img.size(2) - c[0]
# Color
img[0, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
img[1, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
img[2, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
# Prepare image and groundtruth map
inp = HumanAug.crop(imutils.im_to_numpy(img), c.numpy(), s.numpy(), r,
self.inp_res, self.std_size)
inp = imutils.im_to_torch(inp).float()
# inp = self.color_normalize(inp, self.mean, self.std)
pts_aug = HumanAug.TransformPts(pts.numpy(), c.numpy(), s.numpy(), r,
self.out_res, self.std_size)
#idx_indicator = (pts[:, 0] <= 0) | (pts[:, 1] <= 0)
#idx = torch.arange(0, pts.size(0)).long()
#idx = idx[idx_indicator]
#pts_aug[idx, :] = 0
# Generate ground truth
heatmap, pts_aug = HumanPts.pts2heatmap(pts_aug,
[self.out_res, self.out_res],
sigma=1)
heatmap = torch.from_numpy(heatmap).float()
# pts_aug = torch.from_numpy(pts_aug).float()
r = torch.FloatTensor([r])
#normalizer = torch.FloatTensor([normalizer])
if self.is_train:
#print 'inp size: ', inp.size()
#print 'heatmap size: ', heatmap.size()
#print 'c size: ', c.size()
#print 's size: ', s.size()
#print 'r size: ', r.size()
#print 'pts size: ', pts.size()
#print 'normalizer size: ', normalizer.size()
#print 'r: ', r
# if len(r.size()) != 1:
# print 'r: ', r
# if len(c.size()) != 1:
# print 'c: ', c
return inp, heatmap, c, s, r, pts, normalizer
else:
# Meta info
#meta = {'index': index, 'center': c, 'scale': s,
# 'pts': pts, 'tpts': pts_aug}
return inp, heatmap, c, s, r, pts, normalizer, index
def __getitem__(self, index):
if self.is_train:
a = self.anno[self.train[index]]
else:
a = self.anno[self.valid[index]]
img_path = os.path.join(self.img_folder, a['img_paths'])
pts = torch.Tensor(a['joint_self'])
# pts[:, 0:2] -= 1 # Convert pts to zero based
pts = pts[:, 0:2]
# c = torch.Tensor(a['objpos']) - 1
c = torch.Tensor(a['objpos'])
# print c
s = torch.Tensor([a['scale_provided']])
# print s
# exit()
if a['dataset'] == 'MPII':
c[1] = c[1] + 15 * s[0]
s = s * 1.25
normalizer = a['normalizer'] * 0.6
elif a['dataset'] == 'LEEDS':
print 'using lsp data'
s = s * 1.4375
normalizer = torch.dist(pts[2, :], pts[13, :])
else:
print 'no such dataset {}'.format(a['dataset'])
# For single-person pose estimation with a centered/scaled figure
img = imutils.load_image(img_path)
# print img.size()
# exit()
# img = scipy.misc.imread(img_path, mode='RGB') # CxHxW
# img = torch.from_numpy(img)
inp_std, heatmap_std = self.gen_img_heatmap(c.clone(), s.clone(), 0,
img.clone(), pts.clone())
# r = 0
if self.is_train:
s = s * (2 ** (sample_from_bounded_gaussian(self.scale_factor)))
r = sample_from_bounded_gaussian(self.rot_factor)
if np.random.uniform(0, 1, 1) <= 0.6:
r = np.array([0])
# Flip
if np.random.random() <= 0.5:
img = torch.from_numpy(HumanAug.fliplr(img.numpy())).float()
pts = HumanAug.shufflelr(pts, width=img.size(2), dataset='mpii')
c[0] = img.size(2) - c[0]
# Color
img[0, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
img[1, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
img[2, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
# aug image and groundtruth map
inp, heatmap = self.gen_img_heatmap(c.clone(), s.clone(), r,
img.clone(), pts.clone())
r = torch.FloatTensor([r])
return inp_std, inp, heatmap, c, s, r, pts, normalizer, index
else:
# Meta info
#meta = {'index': index, 'center': c, 'scale': s,
# 'pts': pts, 'tpts': pts_aug}
r = torch.FloatTensor([0])
return inp_std, heatmap_std, c, s, r, pts, normalizer, index
def __getitem__(self, index):
if self.is_train:
a = self.anno[self.train[index]]
else:
a = self.anno[self.valid[index]]
img_path = os.path.join(self.img_folder, a['img_paths'])
if a['pts_paths'] == "unknown.xyz":
pts = a['pts']
else:
pts_path = os.path.join(self.img_folder, a['pts_paths'])
if pts_path[-4:] == '.txt':
pts = np.loadtxt(pts_path) # L x 2
else:
pts = a['pts']
pts = np.array(pts)
# Assume all points are visible for a dataset. This is a multiclass
# visibility
visible_multiclass = np.ones(pts.shape[0])
if a['dataset'] == 'aflw_ours' or a['dataset'] == 'cofw_68':
# The pts which are labelled -1 in both x and y are not visible points
self_occluded_landmark = (pts[:, 0] == -1) & (pts[:, 1] == -1)
external_occluded_landmark = (pts[:, 0] < -1) & (pts[:, 1] < -1)
visible_multiclass[self_occluded_landmark] = 0
visible_multiclass[external_occluded_landmark] = 2
# valid landmarks are those which are external occluded and not occluded
valid_landmark = (pts[:, 0] != -1) & (pts[:, 1] != -1)
# The points which are partially occluded have both coordinates as negative but not -1
# Make them positive
pts = np.abs(pts)
# valid_landmark is 0 for to be masked and 1 for not to be masked
# mask is 1 for to be masked and 0 for not to be masked
pts_masked = np.ma.array(pts,
mask=np.column_stack(
(1 - valid_landmark,
1 - valid_landmark)))
pts_mean = np.mean(pts_masked, axis=0)
# Replace -1 by mean of valid landmarks. Otherwise taking min for
# calculating geomteric mean of the box can create issues later.
pts[self_occluded_landmark] = pts_mean.data
scale_mul_factor = 1.1
elif a['dataset'] == "aflw" or a['dataset'] == "wflw":
self_occluded_landmark = (pts[:, 0] <= 0) | (pts[:, 1] <= 0)
valid_landmark = 1 - self_occluded_landmark
visible_multiclass[self_occluded_landmark] = 0
# valid_landmark is 0 for to be masked and 1 for not to be masked
# mask is 1 for to be masked and 0 for not to be masked
pts_masked = np.ma.array(pts,
mask=np.column_stack(
(1 - valid_landmark,
1 - valid_landmark)))
pts_mean = np.mean(pts_masked, axis=0)
# Replace -1 by mean of valid landmarks. Otherwise taking min for
# calculating geomteric mean of the box can create issues later.
pts[self_occluded_landmark] = pts_mean.data
scale_mul_factor = 1.25
else:
scale_mul_factor = 1.1
pts = torch.Tensor(pts) # size is 68*2
s = torch.Tensor([a['scale_provided_det']]) * scale_mul_factor
c = torch.Tensor(a['objpos_det'])
# For single-person pose estimation with a centered/scaled figure
# the image in the original size
img = imutils.load_image(img_path)
r = 0
s_rand = 1
if self.is_train: #data augmentation for training data
s_rand = (1 + sample_from_bounded_gaussian(self.scale_factor / 2.))
s = s * s_rand
r = sample_from_bounded_gaussian(self.rot_factor / 2.)
#print('s shape is ', s.size(), 's is ', s)
#if np.random.uniform(0, 1, 1) <= 0.6:
# r = np.array([0])
if self.use_flipping:
# Flip
if np.random.random() <= 0.5:
img = torch.from_numpy(HumanAug.fliplr(
img.numpy())).float()
pts = HumanAug.shufflelr(pts,
width=img.size(2),
dataset='face')
c[0] = img.size(2) - c[0]
# Color
img[0, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
img[1, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
img[2, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
if self.use_occlusion:
# Apply a random black occlusion
# C x H x W
patch_center_row = randint(1, img.size(1))
patch_center_col = randint(1, img.size(2))
patch_height = randint(1, img.size(1) / 2)
patch_width = randint(1, img.size(2) / 2)
row_min = max(0, patch_center_row - patch_height)
row_max = min(img.size(1), patch_center_row + patch_height)
col_min = max(0, patch_center_col - patch_width)
col_max = min(img.size(2), patch_center_col + patch_width)
img[:, row_min:row_max, col_min:col_max] = 0
# Prepare points first
pts_input_res = HumanAug.TransformPts(pts.numpy(), c.numpy(),
s.numpy(), r, self.inp_res,
self.std_size)
# Some landmark points can go outside after transformation. Determine the
# extra scaling required.
# This can only be done for the training points. For validation, we do
# not know the points location.
if self.is_train and self.keep_pts_inside:
# visible copy takes care of whether point is visible or not.
visible_copy = visible_multiclass.copy()
visible_copy[visible_multiclass > 1] = 1
scale_down = get_ideal_scale(pts_input_res,
self.inp_res,
img_path,
visible=visible_copy)
s = s / scale_down
s_rand = s_rand / scale_down
pts_input_res = HumanAug.TransformPts(pts.numpy(), c.numpy(),
s.numpy(), r, self.inp_res,
self.std_size)
if a['dataset'] == "aflw":
meta_box_size = a['box_size']
# We convert the meta_box size also to the input res. The meta_box
# is not formed by the landmark point but is supplied externally.
# We assume the meta_box as two points [meta_box_size, 0] and [0, 0]
# apply the transformation on top of it
temp = HumanAug.TransformPts(
np.array([[meta_box_size, 0], [0, 0]]), c.numpy(), s.numpy(),
r, self.inp_res, self.std_size)
# Passed as array of 2 x 2
# we only want the transformed distance between the points
meta_box_size_input_res = np.linalg.norm(temp[1] - temp[0])
else:
meta_box_size_input_res = -10 # some invalid number
# pts_input_res is in the size of 256 x 256
# Bring down to 64 x 64 since finally heatmap will be 64 x 64
pts_aug = pts_input_res * (1. * self.out_res / self.inp_res)
# Prepare image
inp = HumanAug.crop(imutils.im_to_numpy(img), c.numpy(), s.numpy(), r,
self.inp_res, self.std_size)
inp_vis = inp
inp = imutils.im_to_torch(inp).float() # 3*256*256
# Generate proxy ground truth heatmap
heatmap, pts_aug = HumanPts.pts2heatmap(pts_aug,
[self.out_res, self.out_res],
sigma=self.sigma)
heatmap = torch.from_numpy(heatmap).float()
heatmap_mask = HumanPts.pts2mask(pts_aug, [self.out_res, self.out_res],
bb=10)
if self.is_train:
return inp, heatmap, pts_input_res, heatmap_mask, s_rand, visible_multiclass, meta_box_size_input_res
else:
return inp, heatmap, pts_input_res, c, s, index, inp_vis, s_rand, visible_multiclass, meta_box_size_input_res
def __getitem__(self, index):
# print('loading image', index)
if self.img_index_list is None:
a = self.anno[self.train[index]]
else:
idx = self.img_index_list[index]
a = self.anno[self.train[idx]]
img_path = os.path.join(self.img_folder, a['img_paths'])
pts = torch.Tensor(a['joint_self'])
# pts[:, 0:2] -= 1 # Convert pts to zero based
pts = pts[:, 0:2]
# c = torch.Tensor(a['objpos']) - 1
c = torch.Tensor(a['objpos'])
# print(c)
s = torch.Tensor([a['scale_provided']])
# r = torch.FloatTensor([0])
# exit()
if a['dataset'] == 'MPII':
c[1] = c[1] + 15 * s[0]
s = s * 1.25
normalizer = a['normalizer'] * 0.6
elif a['dataset'] == 'LEEDS':
print('using lsp data')
s = s * 1.4375
normalizer = torch.dist(pts[2, :], pts[13, :])
else:
print('no such dataset {}'.format(a['dataset']))
# For single-person pose estimation with a centered/scaled figure
img = imutils.load_image(img_path)
if self.img_index_list is None:
s_aug = s * (2**(sample_from_large_gaussian(self.scale_factor)))
r_aug = sample_from_large_gaussian(self.rot_factor)
if np.random.uniform(0, 1, 1) <= 0.6:
r_aug = np.array([0])
else:
gaussian_mean_scale = self.scale_means[
self.scale_index_list[index]]
scale_factor = sample_from_small_gaussian(gaussian_mean_scale,
self.scale_var)
gaussian_mean_rotation = self.rotation_means[
self.rotation_index_list[index]]
r_aug = sample_from_small_gaussian(gaussian_mean_rotation,
self.rotaiton_var)
s_aug = s * (2**scale_factor)
if self.separate_s_r:
img_list = [None] * 2
heatmap_list = [None] * 2
c_list = [c.clone()] * 2
s_list = [s_aug.clone(), s.clone()]
r_list = [torch.FloatTensor([0]), torch.FloatTensor([r_aug])]
grnd_pts_list = [pts.clone(), pts.clone()]
# print('type of normalizaer: ', type(normalizer))
normalizer_list = [normalizer, normalizer]
img_list[0], heatmap_list[0] = self.gen_img_heatmap(
c.clone(), s_aug.clone(), 0, img.clone(), pts.clone())
img_list[1], heatmap_list[1] = self.gen_img_heatmap(
c.clone(), s.clone(), r_aug, img.clone(), pts.clone())
if self.img_index_list is not None:
return img_list, heatmap_list, c_list, s_list,\
r_list, grnd_pts_list, normalizer_list, idx
else:
inp_std, _ = self.gen_img_heatmap(c.clone(), s.clone(), 0,
img.clone(), pts.clone())
return inp_std, img_list, heatmap_list, c_list, s_list, r_list,\
grnd_pts_list, normalizer_list, index
else:
if np.random.random() <= 0.5:
img = torch.from_numpy(HumanAug.fliplr(img.numpy())).float()
pts = HumanAug.shufflelr(pts,
width=img.size(2),
dataset='mpii')
c[0] = img.size(2) - c[0]
# Color
img[0, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
img[1, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
img[2, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
inp, heatmap = self.gen_img_heatmap(c.clone(),
s_aug.clone(), r_aug,
img.clone(), pts.clone())
# if self.separate_s_r is false, then self.img_index_list is not None
# so return idx instead of index
r_aug = torch.FloatTensor([r_aug])
return inp, heatmap, c, s_aug, r_aug, pts, normalizer, idx
shuffle=False,
num_workers=1,
pin_memory=True)
# Visualize some images
for i, (img, _, points, _, s) in enumerate(val_loader):
print(i)
image = img[index].numpy()
pts = points[index].numpy()
plt.figure(figsize=(16, 8))
plt.subplot(121)
plt.imshow(swap_channels(image))
plt_pts(plt, pts)
image = torch.from_numpy(HumanAug.fliplr(img[index].numpy())).float()
pts = HumanAug.shufflelr(points[index],
width=image.size(2),
dataset='face')
plt.subplot(122)
plt.imshow(swap_channels(image))
plt_pts(plt, pts)
plt.show()
plt.close()
if i > 10:
break
val_loader = torch.utils.data.DataLoader(FACE(
"dataset/all_300Wtest_train.json", ".", is_train=True),
batch_size=10,