def generateSampleFace(self, idx):
sf = self.scale_factor
rf = self.rot_factor
main_pts = load_lua(
os.path.join(self.img_folder, 'landmarks_t7',
self.anno[idx].split('_')[0],
self.anno[idx][:-4] + '.t7'))
pts = main_pts[0] if self.pointType == '2D' else main_pts[1]
#pts2 = main_pts[1]
c = torch.Tensor((450 / 2, 450 / 2 + 50))
s = 1.8
img = load_image(
os.path.join(self.img_folder, self.anno[idx].split('_')[0],
self.anno[idx][:-8] + '.jpg'))
r = 0
if self.is_train:
s = s * torch.randn(1).mul_(sf).add_(1).clamp(1 - sf, 1 + sf)[0]
r = torch.randn(1).mul_(rf).clamp(
-2 * rf, 2 * rf)[0] if random.random() <= 0.6 else 0
if random.random() <= 0.5:
img = torch.from_numpy(fliplr(img.numpy())).float()
pts = shufflelr(pts, width=img.size(2), dataset='w300lp')
c[0] = img.size(2) - c[0]
img[0, :, :].mul_(random.uniform(0.7, 1.3)).clamp_(0, 1)
img[1, :, :].mul_(random.uniform(0.7, 1.3)).clamp_(0, 1)
img[2, :, :].mul_(random.uniform(0.7, 1.3)).clamp_(0, 1)
# Prepare image and groundtruth map
inp = HumanAug.crop(imutils.im_to_numpy(img), c.numpy(), s, r, 256,
200)
inp = imutils.im_to_torch(inp).float()
pts_input_res = HumanAug.TransformPts(pts.numpy(), c.numpy(), s, r,
256, 200)
pts_aug = pts_input_res * (1. * 64 / 256)
# Generate ground truth
heatmap, pts_aug = HumanPts.pts2heatmap(pts_aug, [64, 64], sigma=1)
heatmap = torch.from_numpy(heatmap).float()
# inp = crop(img, c, s, [256, 256], rot=r)
# # inp = color_normalize(inp, self.mean, self.std)
# tpts = pts.clone()
# out = torch.zeros(self.nParts, 64, 64)
# for i in range(self.nParts):
# if tpts[i, 0] > 0:
# tpts[i, 0:2] = to_torch(transform(tpts[i, 0:2] + 1, c, s, [64, 64], rot=r))
# out[i] = draw_labelmap(out[i], tpts[i] - 1, sigma=1)
return inp, heatmap, pts, c, s, pts_input_res
def display_imgpts(self, img_batch_list, pts_batch_list, win_id):
for img_batch, pts_batch in itertools.izip(img_batch_list,
pts_batch_list):
img_plot_batch = np.zeros(img_batch.size())
for b in range(0, img_batch.size(0)):
img_np = img_batch[b, :].mul(255.).numpy().astype('uint8')
pts_np = pts_batch[b, :].numpy()
img_pil = Image.fromarray(img_np.transpose([1, 2, 0]), 'RGB')
img_plot = HumanPts.DrawImgPts(img_pil, pts_np)
img_plot = np.asarray(img_plot, dtype='uint8')
img_plot_batch[b, ] = img_plot.transpose([2, 0, 1])
self.vis.images(img_plot_batch,
opts={'title': str(win_id)},
win=win_id)
win_id += 1
def gen_img_heatmap(self, c, s, r, img, pts):
# Prepare image and groundtruth map
# print s[0]/s0[0], r
inp = HumanAug.crop(imutils.im_to_numpy(img), c.numpy(),
s.numpy(), r, self.inp_res, self.std_size)
inp = imutils.im_to_torch(inp).float()
# inp = self.color_normalize(inp, self.mean, self.std)
pts_aug = HumanAug.TransformPts(pts.numpy(), c.numpy(),
s.numpy(), r, self.out_res, self.std_size)
idx_indicator = (pts[:, 0] <= 0) | (pts[:, 1] <= 0)
idx = torch.arange(0, pts.size(0)).long()
idx = idx[idx_indicator]
pts_aug[idx, :] = 0
# Generate ground truth
heatmap, pts_aug = HumanPts.pts2heatmap(pts_aug, [self.out_res, self.out_res], sigma=1)
heatmap = torch.from_numpy(heatmap).float()
# pts_aug = torch.from_numpy(pts_aug).float()
return inp, heatmap
def __getitem__(self, index):
if self.is_train:
a = self.anno[self.train[index]]
else:
a = self.anno[self.valid[index]]
img_path = os.path.join(self.img_folder, a['img_paths'])
pts = torch.Tensor(a['joint_self'])
# pts[:, 0:2] -= 1 # Convert pts to zero based
pts = pts[:, 0:2]
# c = torch.Tensor(a['objpos']) - 1
c = torch.Tensor(a['objpos'])
# print c
s = torch.Tensor([a['scale_provided']])
# print s
# exit()
if a['dataset'] == 'MPII':
c[1] = c[1] + 15 * s[0]
s = s * 1.25
normalizer = a['normalizer'] * 0.6
elif a['dataset'] == 'LEEDS':
print 'using lsp data'
s = s * 1.4375
normalizer = torch.dist(pts[2, :], pts[13, :])
else:
print 'no such dataset {}'.format(a['dataset'])
# For single-person pose estimation with a centered/scaled figure
img = imutils.load_image(img_path)
# print img.size()
# exit()
# img = scipy.misc.imread(img_path, mode='RGB') # CxHxW
# img = torch.from_numpy(img)
r = 0
if self.is_train:
s = s * (2**(sample_from_bounded_gaussian(self.scale_factor)))
r = sample_from_bounded_gaussian(self.rot_factor)
if np.random.uniform(0, 1, 1) <= 0.6:
r = 0
# Flip
if np.random.random() <= 0.5:
img = torch.from_numpy(HumanAug.fliplr(img.numpy())).float()
pts = HumanAug.shufflelr(pts,
width=img.size(2),
dataset='mpii')
c[0] = img.size(2) - c[0]
# Color
img[0, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
img[1, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
img[2, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
# Prepare image and groundtruth map
inp = HumanAug.crop(imutils.im_to_numpy(img), c.numpy(), s.numpy(), r,
self.inp_res, self.std_size)
inp = imutils.im_to_torch(inp).float()
# inp = self.color_normalize(inp, self.mean, self.std)
pts_aug = HumanAug.TransformPts(pts.numpy(), c.numpy(), s.numpy(), r,
self.out_res, self.std_size)
#idx_indicator = (pts[:, 0] <= 0) | (pts[:, 1] <= 0)
#idx = torch.arange(0, pts.size(0)).long()
#idx = idx[idx_indicator]
#pts_aug[idx, :] = 0
# Generate ground truth
heatmap, pts_aug = HumanPts.pts2heatmap(pts_aug,
[self.out_res, self.out_res],
sigma=1)
heatmap = torch.from_numpy(heatmap).float()
# pts_aug = torch.from_numpy(pts_aug).float()
r = torch.FloatTensor([r])
#normalizer = torch.FloatTensor([normalizer])
if self.is_train:
#print 'inp size: ', inp.size()
#print 'heatmap size: ', heatmap.size()
#print 'c size: ', c.size()
#print 's size: ', s.size()
#print 'r size: ', r.size()
#print 'pts size: ', pts.size()
#print 'normalizer size: ', normalizer.size()
#print 'r: ', r
# if len(r.size()) != 1:
# print 'r: ', r
# if len(c.size()) != 1:
# print 'c: ', c
return inp, heatmap, c, s, r, pts, normalizer
else:
# Meta info
#meta = {'index': index, 'center': c, 'scale': s,
# 'pts': pts, 'tpts': pts_aug}
return inp, heatmap, c, s, r, pts, normalizer, index
def __getitem__(self, index):
if self.is_train:
a = self.anno[self.train[index]]
else:
a = self.anno[self.valid[index]]
img_path = os.path.join(self.img_folder, a['img_paths'])
pts_path = os.path.join(self.img_folder, a['pts_paths'])
skip_pts = [33, 36, 39, 42, 45, 48, 51, 54, 57]
if pts_path[-4:] == '.txt':
pts = np.loadtxt(pts_path) # L x 2
#pts = pts[skip_pts, :]
elif pts_path[-4:] == '.pts':
pts = FacePts.Pts2Lmk(pts_path) # L x 2
#pts = pts[skip_pts, :]
#print(pts)
pts = torch.Tensor(pts)
assert torch.sum(pts - torch.Tensor(a['pts'])) == 0
s = torch.Tensor([a['scale_provided_det']]) * 1.1
c = torch.Tensor(a['objpos_det'])
# For single-person pose estimation with a centered/scaled figure
img = imutils.load_image(img_path)
# print img.size()
# exit()
# img = scipy.misc.imread(img_path, mode='RGB') # CxHxW
# img = torch.from_numpy(img)
r = 0
if self.is_train:
s = s * (2**(sample_from_bounded_gaussian(self.scale_factor)))
r = sample_from_bounded_gaussian(self.rot_factor)
if np.random.uniform(0, 1, 1) <= 0.6:
r = np.array([0])
# Flip
#if np.random.random() <= 0.5:
# img = torch.from_numpy(HumanAug.fliplr(img.numpy())).float()
# pts = HumanAug.shufflelr(pts, width=img.size(2), dataset='face')
# c[0] = img.size(2) - c[0]
# Color
img[0, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
img[1, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
img[2, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
# Prepare image and groundtruth map
inp = HumanAug.crop(imutils.im_to_numpy(img), c.numpy(), s.numpy(), r,
self.inp_res, self.std_size)
inp = imutils.im_to_torch(inp).float()
# inp = self.color_normalize(inp, self.mean, self.std)
# pts_aug = HumanAug.TransformPts(pts.numpy(), c.numpy(),
# s.numpy(), r, self.out_res, self.std_size)
pts_input_res = HumanAug.TransformPts(pts.numpy(), c.numpy(),
s.numpy(), r, self.inp_res,
self.std_size)
pts_aug = pts_input_res * (1. * self.out_res / self.inp_res)
#check_res = pts_input_res - pts
#print('diff.... -> {}'.format(check_res))
# Generate ground truth
heatmap, pts_aug = HumanPts.pts2heatmap(pts_aug,
[self.out_res, self.out_res],
sigma=1)
heatmap = torch.from_numpy(heatmap).float()
# pts_aug = torch.from_numpy(pts_aug).float()
if self.is_train:
return inp, heatmap, pts_input_res
else:
# Meta info
#meta = {'index': index, 'center': c, 'scale': s,
# 'pts': pts, 'tpts': pts_aug}
return inp, heatmap, pts, index, c, s, img_path
def __getitem__(self, index):
if self.is_train:
a = self.anno[self.train[index]]
else:
a = self.anno[self.valid[index]]
img_path = os.path.join(self.img_folder, a['img_paths'])
if a['pts_paths'] == "unknown.xyz":
pts = a['pts']
else:
pts_path = os.path.join(self.img_folder, a['pts_paths'])
if pts_path[-4:] == '.txt':
pts = np.loadtxt(pts_path) # L x 2
else:
pts = a['pts']
pts = np.array(pts)
# Assume all points are visible for a dataset. This is a multiclass
# visibility
visible_multiclass = np.ones(pts.shape[0])
if a['dataset'] == 'aflw_ours' or a['dataset'] == 'cofw_68':
# The pts which are labelled -1 in both x and y are not visible points
self_occluded_landmark = (pts[:, 0] == -1) & (pts[:, 1] == -1)
external_occluded_landmark = (pts[:, 0] < -1) & (pts[:, 1] < -1)
visible_multiclass[self_occluded_landmark] = 0
visible_multiclass[external_occluded_landmark] = 2
# valid landmarks are those which are external occluded and not occluded
valid_landmark = (pts[:, 0] != -1) & (pts[:, 1] != -1)
# The points which are partially occluded have both coordinates as negative but not -1
# Make them positive
pts = np.abs(pts)
# valid_landmark is 0 for to be masked and 1 for not to be masked
# mask is 1 for to be masked and 0 for not to be masked
pts_masked = np.ma.array(pts,
mask=np.column_stack(
(1 - valid_landmark,
1 - valid_landmark)))
pts_mean = np.mean(pts_masked, axis=0)
# Replace -1 by mean of valid landmarks. Otherwise taking min for
# calculating geomteric mean of the box can create issues later.
pts[self_occluded_landmark] = pts_mean.data
scale_mul_factor = 1.1
elif a['dataset'] == "aflw" or a['dataset'] == "wflw":
self_occluded_landmark = (pts[:, 0] <= 0) | (pts[:, 1] <= 0)
valid_landmark = 1 - self_occluded_landmark
visible_multiclass[self_occluded_landmark] = 0
# valid_landmark is 0 for to be masked and 1 for not to be masked
# mask is 1 for to be masked and 0 for not to be masked
pts_masked = np.ma.array(pts,
mask=np.column_stack(
(1 - valid_landmark,
1 - valid_landmark)))
pts_mean = np.mean(pts_masked, axis=0)
# Replace -1 by mean of valid landmarks. Otherwise taking min for
# calculating geomteric mean of the box can create issues later.
pts[self_occluded_landmark] = pts_mean.data
scale_mul_factor = 1.25
else:
scale_mul_factor = 1.1
pts = torch.Tensor(pts) # size is 68*2
s = torch.Tensor([a['scale_provided_det']]) * scale_mul_factor
c = torch.Tensor(a['objpos_det'])
# For single-person pose estimation with a centered/scaled figure
# the image in the original size
img = imutils.load_image(img_path)
r = 0
s_rand = 1
if self.is_train: #data augmentation for training data
s_rand = (1 + sample_from_bounded_gaussian(self.scale_factor / 2.))
s = s * s_rand
r = sample_from_bounded_gaussian(self.rot_factor / 2.)
#print('s shape is ', s.size(), 's is ', s)
#if np.random.uniform(0, 1, 1) <= 0.6:
# r = np.array([0])
if self.use_flipping:
# Flip
if np.random.random() <= 0.5:
img = torch.from_numpy(HumanAug.fliplr(
img.numpy())).float()
pts = HumanAug.shufflelr(pts,
width=img.size(2),
dataset='face')
c[0] = img.size(2) - c[0]
# Color
img[0, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
img[1, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
img[2, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
if self.use_occlusion:
# Apply a random black occlusion
# C x H x W
patch_center_row = randint(1, img.size(1))
patch_center_col = randint(1, img.size(2))
patch_height = randint(1, img.size(1) / 2)
patch_width = randint(1, img.size(2) / 2)
row_min = max(0, patch_center_row - patch_height)
row_max = min(img.size(1), patch_center_row + patch_height)
col_min = max(0, patch_center_col - patch_width)
col_max = min(img.size(2), patch_center_col + patch_width)
img[:, row_min:row_max, col_min:col_max] = 0
# Prepare points first
pts_input_res = HumanAug.TransformPts(pts.numpy(), c.numpy(),
s.numpy(), r, self.inp_res,
self.std_size)
# Some landmark points can go outside after transformation. Determine the
# extra scaling required.
# This can only be done for the training points. For validation, we do
# not know the points location.
if self.is_train and self.keep_pts_inside:
# visible copy takes care of whether point is visible or not.
visible_copy = visible_multiclass.copy()
visible_copy[visible_multiclass > 1] = 1
scale_down = get_ideal_scale(pts_input_res,
self.inp_res,
img_path,
visible=visible_copy)
s = s / scale_down
s_rand = s_rand / scale_down
pts_input_res = HumanAug.TransformPts(pts.numpy(), c.numpy(),
s.numpy(), r, self.inp_res,
self.std_size)
if a['dataset'] == "aflw":
meta_box_size = a['box_size']
# We convert the meta_box size also to the input res. The meta_box
# is not formed by the landmark point but is supplied externally.
# We assume the meta_box as two points [meta_box_size, 0] and [0, 0]
# apply the transformation on top of it
temp = HumanAug.TransformPts(
np.array([[meta_box_size, 0], [0, 0]]), c.numpy(), s.numpy(),
r, self.inp_res, self.std_size)
# Passed as array of 2 x 2
# we only want the transformed distance between the points
meta_box_size_input_res = np.linalg.norm(temp[1] - temp[0])
else:
meta_box_size_input_res = -10 # some invalid number
# pts_input_res is in the size of 256 x 256
# Bring down to 64 x 64 since finally heatmap will be 64 x 64
pts_aug = pts_input_res * (1. * self.out_res / self.inp_res)
# Prepare image
inp = HumanAug.crop(imutils.im_to_numpy(img), c.numpy(), s.numpy(), r,
self.inp_res, self.std_size)
inp_vis = inp
inp = imutils.im_to_torch(inp).float() # 3*256*256
# Generate proxy ground truth heatmap
heatmap, pts_aug = HumanPts.pts2heatmap(pts_aug,
[self.out_res, self.out_res],
sigma=self.sigma)
heatmap = torch.from_numpy(heatmap).float()
heatmap_mask = HumanPts.pts2mask(pts_aug, [self.out_res, self.out_res],
bb=10)
if self.is_train:
return inp, heatmap, pts_input_res, heatmap_mask, s_rand, visible_multiclass, meta_box_size_input_res
else:
return inp, heatmap, pts_input_res, c, s, index, inp_vis, s_rand, visible_multiclass, meta_box_size_input_res
def generateSampleFace(self, idx):
sf = self.scale_factor
rf = self.rot_factor
#print('Filename -->{}'.format(self.anno[idx][:-4] + '.jpg'))
main_pts = sio.loadmat(self.anno[idx])
pts = main_pts['pt3d_68'][0:2, :].transpose()
#print(pts.dtype)
pts = np.float32(pts)
pts = torch.from_numpy(pts)
#print('pts -> {}'.format(pts))
pts = torch.clamp(pts, min=0)
mins_ = torch.min(pts, 0)[0].view(2) # min vals
maxs_ = torch.max(pts, 0)[0].view(2) # max vals
c = torch.FloatTensor((maxs_[0] - (maxs_[0] - mins_[0]) / 2,
maxs_[1] - (maxs_[1] - mins_[1]) / 2))
#print('min Values format -> {}'.format(mins_.dtype))
#print('max Values format -> {}'.format(maxs_.dtype))
c[1] -= ((maxs_[1] - mins_[1]) * 0.12)
s = (maxs_[0] - mins_[0] + maxs_[1] - mins_[1]) / 195
img = load_image(self.anno[idx][:-4] + '.jpg')
r = 0
if self.is_train:
s = s * torch.randn(1).mul_(sf).add_(1).clamp(1 - sf, 1 + sf)[0]
r = torch.randn(1).mul_(rf).clamp(
-2 * rf, 2 * rf)[0] if random.random() <= 0.6 else 0
if random.random() <= 0.5:
img = torch.from_numpy(fliplr(img.numpy())).float()
pts = shufflelr(pts, width=img.size(2), dataset='aflw2000')
c[0] = img.size(2) - c[0]
img[0, :, :].mul_(random.uniform(0.7, 1.3)).clamp_(0, 1)
img[1, :, :].mul_(random.uniform(0.7, 1.3)).clamp_(0, 1)
img[2, :, :].mul_(random.uniform(0.7, 1.3)).clamp_(0, 1)
# Prepare image and groundtruth map
inp = HumanAug.crop(imutils.im_to_numpy(img), c.numpy(), s, r, 256,
200)
inp = imutils.im_to_torch(inp).float()
pts_input_res = HumanAug.TransformPts(pts.numpy(), c.numpy(), s, r,
256, 200)
pts_aug = pts_input_res * (1. * 64 / 256)
# Generate ground truth
heatmap, pts_aug = HumanPts.pts2heatmap(pts_aug, [64, 64], sigma=1)
heatmap = torch.from_numpy(heatmap).float()
# inp = crop(img, c, s, [256, 256], rot=r)
# # inp = color_normalize(inp, self.mean, self.std)
# tpts = pts.clone()
# out = torch.zeros(self.nParts, 64, 64)
# for i in range(self.nParts):
# if tpts[i, 0] > 0:
# tpts[i, 0:2] = to_torch(transform(tpts[i, 0:2] + 1, c, s, [64, 64], rot=r))
# out[i] = draw_labelmap(out[i], tpts[i] - 1, sigma=1)
return inp, heatmap, pts, c, s, pts_input_res