def generateSampleFace(self, idx):
sf = self.scale_factor
rf = self.rot_factor
main_pts = load_lua(
os.path.join(self.img_folder, 'landmarks_t7',
self.anno[idx].split('_')[0],
self.anno[idx][:-4] + '.t7'))
pts = main_pts[0] if self.pointType == '2D' else main_pts[1]
#pts2 = main_pts[1]
c = torch.Tensor((450 / 2, 450 / 2 + 50))
s = 1.8
img = load_image(
os.path.join(self.img_folder, self.anno[idx].split('_')[0],
self.anno[idx][:-8] + '.jpg'))
r = 0
if self.is_train:
s = s * torch.randn(1).mul_(sf).add_(1).clamp(1 - sf, 1 + sf)[0]
r = torch.randn(1).mul_(rf).clamp(
-2 * rf, 2 * rf)[0] if random.random() <= 0.6 else 0
if random.random() <= 0.5:
img = torch.from_numpy(fliplr(img.numpy())).float()
pts = shufflelr(pts, width=img.size(2), dataset='w300lp')
c[0] = img.size(2) - c[0]
img[0, :, :].mul_(random.uniform(0.7, 1.3)).clamp_(0, 1)
img[1, :, :].mul_(random.uniform(0.7, 1.3)).clamp_(0, 1)
img[2, :, :].mul_(random.uniform(0.7, 1.3)).clamp_(0, 1)
# Prepare image and groundtruth map
inp = HumanAug.crop(imutils.im_to_numpy(img), c.numpy(), s, r, 256,
200)
inp = imutils.im_to_torch(inp).float()
pts_input_res = HumanAug.TransformPts(pts.numpy(), c.numpy(), s, r,
256, 200)
pts_aug = pts_input_res * (1. * 64 / 256)
# Generate ground truth
heatmap, pts_aug = HumanPts.pts2heatmap(pts_aug, [64, 64], sigma=1)
heatmap = torch.from_numpy(heatmap).float()
# inp = crop(img, c, s, [256, 256], rot=r)
# # inp = color_normalize(inp, self.mean, self.std)
# tpts = pts.clone()
# out = torch.zeros(self.nParts, 64, 64)
# for i in range(self.nParts):
# if tpts[i, 0] > 0:
# tpts[i, 0:2] = to_torch(transform(tpts[i, 0:2] + 1, c, s, [64, 64], rot=r))
# out[i] = draw_labelmap(out[i], tpts[i] - 1, sigma=1)
return inp, heatmap, pts, c, s, pts_input_res
def gen_img_heatmap(self, c, s, r, img, pts):
# Prepare image and groundtruth map
# print s[0]/s0[0], r
inp = HumanAug.crop(imutils.im_to_numpy(img), c.numpy(),
s.numpy(), r, self.inp_res, self.std_size)
inp = imutils.im_to_torch(inp).float()
# inp = self.color_normalize(inp, self.mean, self.std)
pts_aug = HumanAug.TransformPts(pts.numpy(), c.numpy(),
s.numpy(), r, self.out_res, self.std_size)
idx_indicator = (pts[:, 0] <= 0) | (pts[:, 1] <= 0)
idx = torch.arange(0, pts.size(0)).long()
idx = idx[idx_indicator]
pts_aug[idx, :] = 0
# Generate ground truth
heatmap, pts_aug = HumanPts.pts2heatmap(pts_aug, [self.out_res, self.out_res], sigma=1)
heatmap = torch.from_numpy(heatmap).float()
# pts_aug = torch.from_numpy(pts_aug).float()
return inp, heatmap
def __getitem__(self, index):
# print('loading image', index)
if self.is_train:
a = self.anno[self.train[index]]
else:
a = self.anno[self.valid[index]]
img_path = os.path.join(self.img_folder, a['img_paths'])
pts = torch.Tensor(a['joint_self'])
# pts[:, 0:2] -= 1 # Convert pts to zero based
pts = pts[:, 0:2]
# c = torch.Tensor(a['objpos']) - 1
c = torch.Tensor(a['objpos'])
# print(c)
s = torch.Tensor([a['scale_provided']])
# exit()
if a['dataset'] == 'MPII':
c[1] = c[1] + 15 * s[0]
s = s * 1.25
normalizer = a['normalizer'] * 0.6
elif a['dataset'] == 'LEEDS':
print('using lsp data')
s = s * 1.4375
normalizer = torch.dist(pts[2, :], pts[13, :])
else:
print('no such dataset {}'.format(a['dataset']))
# For single-person pose estimation with a centered/scaled figure
img = imutils.load_image(img_path)
# img = Image.open(img_path)
inp = HumanAug.crop(imutils.im_to_numpy(img), c.numpy(), s.numpy(), 0,
self.inp_res, self.std_size)
inp = imutils.im_to_torch(inp).float()
# inp = self.color_normalize(inp, self.mean, self.std)
# pts_aug = HumanAug.TransformPts(pts.numpy(), c.numpy(),
# s.numpy(), 0, self.out_res, self.std_size)
#
# # Generate ground truth
# heatmap, pts_aug = HumanPts.pts2heatmap(pts_aug, [self.out_res, self.out_res], sigma=1)
# heatmap = torch.from_numpy(heatmap).float()
tmp_scale_distri = self.grnd_scale_distri[
index] / self.grnd_scale_distri[index].sum()
tmp_rot_distri = self.grnd_rotation_distri[
index] / self.grnd_rotation_distri[index].sum()
return inp, tmp_scale_distri, tmp_rot_distri, index
def validate(val_loader, net, epoch, visualizer, idx, num_classes):
batch_time = AverageMeter()
losses_det = AverageMeter()
losses = AverageMeter()
pckhs = AverageMeter()
pckhs_origin_res = AverageMeter()
img_batch_list = []
pts_batch_list = []
# predictions
predictions = torch.Tensor(val_loader.dataset.__len__(), num_classes, 2)
# switch to evaluate mode
net.eval()
end = time.time()
for i, (img, heatmap, center, scale, rot, grnd_pts, normalizer,
index) in enumerate(val_loader):
# input and groundtruth
input_var = torch.autograd.Variable(img, volatile=True)
heatmap = heatmap.cuda(async=True)
target_var = torch.autograd.Variable(heatmap)
# output and loss
#output1, output2 = net(input_var)
#loss = (output1 - target_var) ** 2 + (output2 - target_var) ** 2
output1 = net(input_var)
loss = 0
for per_out in output1:
tmp_loss = (per_out - target_var)**2
loss = loss + tmp_loss.sum() / tmp_loss.numel()
# flipping the image
img_flip = img.numpy()[:, :, :, ::-1].copy()
img_flip = torch.from_numpy(img_flip)
input_var = torch.autograd.Variable(img_flip, volatile=True)
#output11, output22 = net(input_var)
output2 = net(input_var)
output2 = HumanAug.flip_channels(output2[-1].cpu().data)
output2 = HumanAug.shuffle_channels_for_horizontal_flipping(output2)
output = (output1[-1].cpu().data + output2) / 2
# calculate measure
# pred_pts = HumanPts.heatmap2pts(output) # b x L x 2
# pts = HumanPts.heatmap2pts(target_var.cpu().data)
# pckh = HumanAcc.approx_PCKh(pred_pts, pts, idx, heatmap.size(3)) # b -> 1
pckh = Evaluation.accuracy(output, target_var.data.cpu(), idx)
pckhs.update(pckh[0])
pckh_origin_res = Evaluation.accuracy_origin_res(
output, center, scale, [64, 64], grnd_pts, normalizer, rot)
pckhs_origin_res.update(pckh_origin_res[0])
"""measure elapsed time"""
batch_time.update(time.time() - end)
end = time.time()
# print(log)
losses.update(loss.data[0])
loss_dict = OrderedDict([('loss', losses.avg), ('pckh', pckhs.avg),
('pckh_origin_res', pckhs_origin_res.avg)])
visualizer.print_log(epoch, i, len(val_loader), value1=loss_dict)
# img_batch_list.append(img)
# pts_batch_list.append(pred_pts*4.)
# preds = Evaluation.final_preds(output, meta['center'], meta['scale'], [64, 64])
# for n in range(output.size(0)):
# predictions[meta['index'][n], :, :] = preds[n, :, :]
preds = Evaluation.final_preds(output, center, scale, [64, 64], rot)
for n in range(output.size(0)):
predictions[index[n], :, :] = preds[n, :, :]
# if i == 2:
# break
# return losses.avg, pckhs.avg, img_batch_list, pts_batch_list
return losses.avg, pckhs_origin_res.avg, predictions
def __getitem__(self, index):
if self.is_train:
a = self.anno[self.train[index]]
else:
a = self.anno[self.valid[index]]
img_path = os.path.join(self.img_folder, a['img_paths'])
pts = torch.Tensor(a['joint_self'])
# pts[:, 0:2] -= 1 # Convert pts to zero based
pts = pts[:, 0:2]
# c = torch.Tensor(a['objpos']) - 1
c = torch.Tensor(a['objpos'])
# print c
s = torch.Tensor([a['scale_provided']])
# print s
# exit()
if a['dataset'] == 'MPII':
c[1] = c[1] + 15 * s[0]
s = s * 1.25
normalizer = a['normalizer'] * 0.6
elif a['dataset'] == 'LEEDS':
print 'using lsp data'
s = s * 1.4375
normalizer = torch.dist(pts[2, :], pts[13, :])
else:
print 'no such dataset {}'.format(a['dataset'])
# For single-person pose estimation with a centered/scaled figure
img = imutils.load_image(img_path)
# print img.size()
# exit()
# img = scipy.misc.imread(img_path, mode='RGB') # CxHxW
# img = torch.from_numpy(img)
r = 0
if self.is_train:
s = s * (2**(sample_from_bounded_gaussian(self.scale_factor)))
r = sample_from_bounded_gaussian(self.rot_factor)
if np.random.uniform(0, 1, 1) <= 0.6:
r = 0
# Flip
if np.random.random() <= 0.5:
img = torch.from_numpy(HumanAug.fliplr(img.numpy())).float()
pts = HumanAug.shufflelr(pts,
width=img.size(2),
dataset='mpii')
c[0] = img.size(2) - c[0]
# Color
img[0, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
img[1, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
img[2, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
# Prepare image and groundtruth map
inp = HumanAug.crop(imutils.im_to_numpy(img), c.numpy(), s.numpy(), r,
self.inp_res, self.std_size)
inp = imutils.im_to_torch(inp).float()
# inp = self.color_normalize(inp, self.mean, self.std)
pts_aug = HumanAug.TransformPts(pts.numpy(), c.numpy(), s.numpy(), r,
self.out_res, self.std_size)
#idx_indicator = (pts[:, 0] <= 0) | (pts[:, 1] <= 0)
#idx = torch.arange(0, pts.size(0)).long()
#idx = idx[idx_indicator]
#pts_aug[idx, :] = 0
# Generate ground truth
heatmap, pts_aug = HumanPts.pts2heatmap(pts_aug,
[self.out_res, self.out_res],
sigma=1)
heatmap = torch.from_numpy(heatmap).float()
# pts_aug = torch.from_numpy(pts_aug).float()
r = torch.FloatTensor([r])
#normalizer = torch.FloatTensor([normalizer])
if self.is_train:
#print 'inp size: ', inp.size()
#print 'heatmap size: ', heatmap.size()
#print 'c size: ', c.size()
#print 's size: ', s.size()
#print 'r size: ', r.size()
#print 'pts size: ', pts.size()
#print 'normalizer size: ', normalizer.size()
#print 'r: ', r
# if len(r.size()) != 1:
# print 'r: ', r
# if len(c.size()) != 1:
# print 'c: ', c
return inp, heatmap, c, s, r, pts, normalizer
else:
# Meta info
#meta = {'index': index, 'center': c, 'scale': s,
# 'pts': pts, 'tpts': pts_aug}
return inp, heatmap, c, s, r, pts, normalizer, index
def __getitem__(self, index):
if self.is_train:
a = self.anno[self.train[index]]
else:
a = self.anno[self.valid[index]]
img_path = os.path.join(self.img_folder, a['img_paths'])
pts_path = os.path.join(self.img_folder, a['pts_paths'])
skip_pts = [33, 36, 39, 42, 45, 48, 51, 54, 57]
if pts_path[-4:] == '.txt':
pts = np.loadtxt(pts_path) # L x 2
#pts = pts[skip_pts, :]
elif pts_path[-4:] == '.pts':
pts = FacePts.Pts2Lmk(pts_path) # L x 2
#pts = pts[skip_pts, :]
#print(pts)
pts = torch.Tensor(pts)
assert torch.sum(pts - torch.Tensor(a['pts'])) == 0
s = torch.Tensor([a['scale_provided_det']]) * 1.1
c = torch.Tensor(a['objpos_det'])
# For single-person pose estimation with a centered/scaled figure
img = imutils.load_image(img_path)
# print img.size()
# exit()
# img = scipy.misc.imread(img_path, mode='RGB') # CxHxW
# img = torch.from_numpy(img)
r = 0
if self.is_train:
s = s * (2**(sample_from_bounded_gaussian(self.scale_factor)))
r = sample_from_bounded_gaussian(self.rot_factor)
if np.random.uniform(0, 1, 1) <= 0.6:
r = np.array([0])
# Flip
#if np.random.random() <= 0.5:
# img = torch.from_numpy(HumanAug.fliplr(img.numpy())).float()
# pts = HumanAug.shufflelr(pts, width=img.size(2), dataset='face')
# c[0] = img.size(2) - c[0]
# Color
img[0, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
img[1, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
img[2, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
# Prepare image and groundtruth map
inp = HumanAug.crop(imutils.im_to_numpy(img), c.numpy(), s.numpy(), r,
self.inp_res, self.std_size)
inp = imutils.im_to_torch(inp).float()
# inp = self.color_normalize(inp, self.mean, self.std)
# pts_aug = HumanAug.TransformPts(pts.numpy(), c.numpy(),
# s.numpy(), r, self.out_res, self.std_size)
pts_input_res = HumanAug.TransformPts(pts.numpy(), c.numpy(),
s.numpy(), r, self.inp_res,
self.std_size)
pts_aug = pts_input_res * (1. * self.out_res / self.inp_res)
#check_res = pts_input_res - pts
#print('diff.... -> {}'.format(check_res))
# Generate ground truth
heatmap, pts_aug = HumanPts.pts2heatmap(pts_aug,
[self.out_res, self.out_res],
sigma=1)
heatmap = torch.from_numpy(heatmap).float()
# pts_aug = torch.from_numpy(pts_aug).float()
if self.is_train:
return inp, heatmap, pts_input_res
else:
# Meta info
#meta = {'index': index, 'center': c, 'scale': s,
# 'pts': pts, 'tpts': pts_aug}
return inp, heatmap, pts, index, c, s, img_path
def __getitem__(self, index):
if self.is_train:
a = self.anno[self.train[index]]
else:
a = self.anno[self.valid[index]]
img_path = os.path.join(self.img_folder, a['img_paths'])
pts = torch.Tensor(a['joint_self'])
# pts[:, 0:2] -= 1 # Convert pts to zero based
pts = pts[:, 0:2]
# c = torch.Tensor(a['objpos']) - 1
c = torch.Tensor(a['objpos'])
# print c
s = torch.Tensor([a['scale_provided']])
# print s
# exit()
if a['dataset'] == 'MPII':
c[1] = c[1] + 15 * s[0]
s = s * 1.25
normalizer = a['normalizer'] * 0.6
elif a['dataset'] == 'LEEDS':
print 'using lsp data'
s = s * 1.4375
normalizer = torch.dist(pts[2, :], pts[13, :])
else:
print 'no such dataset {}'.format(a['dataset'])
# For single-person pose estimation with a centered/scaled figure
img = imutils.load_image(img_path)
# print img.size()
# exit()
# img = scipy.misc.imread(img_path, mode='RGB') # CxHxW
# img = torch.from_numpy(img)
inp_std, heatmap_std = self.gen_img_heatmap(c.clone(), s.clone(), 0,
img.clone(), pts.clone())
# r = 0
if self.is_train:
s = s * (2 ** (sample_from_bounded_gaussian(self.scale_factor)))
r = sample_from_bounded_gaussian(self.rot_factor)
if np.random.uniform(0, 1, 1) <= 0.6:
r = np.array([0])
# Flip
if np.random.random() <= 0.5:
img = torch.from_numpy(HumanAug.fliplr(img.numpy())).float()
pts = HumanAug.shufflelr(pts, width=img.size(2), dataset='mpii')
c[0] = img.size(2) - c[0]
# Color
img[0, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
img[1, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
img[2, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
# aug image and groundtruth map
inp, heatmap = self.gen_img_heatmap(c.clone(), s.clone(), r,
img.clone(), pts.clone())
r = torch.FloatTensor([r])
return inp_std, inp, heatmap, c, s, r, pts, normalizer, index
else:
# Meta info
#meta = {'index': index, 'center': c, 'scale': s,
# 'pts': pts, 'tpts': pts_aug}
r = torch.FloatTensor([0])
return inp_std, heatmap_std, c, s, r, pts, normalizer, index
def __getitem__(self, index):
# print('loading image', index)
if self.img_index_list is None:
a = self.anno[self.train[index]]
else:
idx = self.img_index_list[index]
a = self.anno[self.train[idx]]
img_path = os.path.join(self.img_folder, a['img_paths'])
pts = torch.Tensor(a['joint_self'])
# pts[:, 0:2] -= 1 # Convert pts to zero based
pts = pts[:, 0:2]
# c = torch.Tensor(a['objpos']) - 1
c = torch.Tensor(a['objpos'])
# print(c)
s = torch.Tensor([a['scale_provided']])
# r = torch.FloatTensor([0])
# exit()
if a['dataset'] == 'MPII':
c[1] = c[1] + 15 * s[0]
s = s * 1.25
normalizer = a['normalizer'] * 0.6
elif a['dataset'] == 'LEEDS':
print('using lsp data')
s = s * 1.4375
normalizer = torch.dist(pts[2, :], pts[13, :])
else:
print('no such dataset {}'.format(a['dataset']))
# For single-person pose estimation with a centered/scaled figure
img = imutils.load_image(img_path)
if self.img_index_list is None:
s_aug = s * (2**(sample_from_large_gaussian(self.scale_factor)))
r_aug = sample_from_large_gaussian(self.rot_factor)
if np.random.uniform(0, 1, 1) <= 0.6:
r_aug = np.array([0])
else:
gaussian_mean_scale = self.scale_means[
self.scale_index_list[index]]
scale_factor = sample_from_small_gaussian(gaussian_mean_scale,
self.scale_var)
gaussian_mean_rotation = self.rotation_means[
self.rotation_index_list[index]]
r_aug = sample_from_small_gaussian(gaussian_mean_rotation,
self.rotaiton_var)
s_aug = s * (2**scale_factor)
if self.separate_s_r:
img_list = [None] * 2
heatmap_list = [None] * 2
c_list = [c.clone()] * 2
s_list = [s_aug.clone(), s.clone()]
r_list = [torch.FloatTensor([0]), torch.FloatTensor([r_aug])]
grnd_pts_list = [pts.clone(), pts.clone()]
# print('type of normalizaer: ', type(normalizer))
normalizer_list = [normalizer, normalizer]
img_list[0], heatmap_list[0] = self.gen_img_heatmap(
c.clone(), s_aug.clone(), 0, img.clone(), pts.clone())
img_list[1], heatmap_list[1] = self.gen_img_heatmap(
c.clone(), s.clone(), r_aug, img.clone(), pts.clone())
if self.img_index_list is not None:
return img_list, heatmap_list, c_list, s_list,\
r_list, grnd_pts_list, normalizer_list, idx
else:
inp_std, _ = self.gen_img_heatmap(c.clone(), s.clone(), 0,
img.clone(), pts.clone())
return inp_std, img_list, heatmap_list, c_list, s_list, r_list,\
grnd_pts_list, normalizer_list, index
else:
if np.random.random() <= 0.5:
img = torch.from_numpy(HumanAug.fliplr(img.numpy())).float()
pts = HumanAug.shufflelr(pts,
width=img.size(2),
dataset='mpii')
c[0] = img.size(2) - c[0]
# Color
img[0, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
img[1, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
img[2, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
inp, heatmap = self.gen_img_heatmap(c.clone(),
s_aug.clone(), r_aug,
img.clone(), pts.clone())
# if self.separate_s_r is false, then self.img_index_list is not None
# so return idx instead of index
r_aug = torch.FloatTensor([r_aug])
return inp, heatmap, c, s_aug, r_aug, pts, normalizer, idx
def __getitem__(self, index):
if self.is_train:
a = self.anno[self.train[index]]
else:
a = self.anno[self.valid[index]]
img_path = os.path.join(self.img_folder, a['img_paths'])
if a['pts_paths'] == "unknown.xyz":
pts = a['pts']
else:
pts_path = os.path.join(self.img_folder, a['pts_paths'])
if pts_path[-4:] == '.txt':
pts = np.loadtxt(pts_path) # L x 2
else:
pts = a['pts']
pts = np.array(pts)
# Assume all points are visible for a dataset. This is a multiclass
# visibility
visible_multiclass = np.ones(pts.shape[0])
if a['dataset'] == 'aflw_ours' or a['dataset'] == 'cofw_68':
# The pts which are labelled -1 in both x and y are not visible points
self_occluded_landmark = (pts[:, 0] == -1) & (pts[:, 1] == -1)
external_occluded_landmark = (pts[:, 0] < -1) & (pts[:, 1] < -1)
visible_multiclass[self_occluded_landmark] = 0
visible_multiclass[external_occluded_landmark] = 2
# valid landmarks are those which are external occluded and not occluded
valid_landmark = (pts[:, 0] != -1) & (pts[:, 1] != -1)
# The points which are partially occluded have both coordinates as negative but not -1
# Make them positive
pts = np.abs(pts)
# valid_landmark is 0 for to be masked and 1 for not to be masked
# mask is 1 for to be masked and 0 for not to be masked
pts_masked = np.ma.array(pts,
mask=np.column_stack(
(1 - valid_landmark,
1 - valid_landmark)))
pts_mean = np.mean(pts_masked, axis=0)
# Replace -1 by mean of valid landmarks. Otherwise taking min for
# calculating geomteric mean of the box can create issues later.
pts[self_occluded_landmark] = pts_mean.data
scale_mul_factor = 1.1
elif a['dataset'] == "aflw" or a['dataset'] == "wflw":
self_occluded_landmark = (pts[:, 0] <= 0) | (pts[:, 1] <= 0)
valid_landmark = 1 - self_occluded_landmark
visible_multiclass[self_occluded_landmark] = 0
# valid_landmark is 0 for to be masked and 1 for not to be masked
# mask is 1 for to be masked and 0 for not to be masked
pts_masked = np.ma.array(pts,
mask=np.column_stack(
(1 - valid_landmark,
1 - valid_landmark)))
pts_mean = np.mean(pts_masked, axis=0)
# Replace -1 by mean of valid landmarks. Otherwise taking min for
# calculating geomteric mean of the box can create issues later.
pts[self_occluded_landmark] = pts_mean.data
scale_mul_factor = 1.25
else:
scale_mul_factor = 1.1
pts = torch.Tensor(pts) # size is 68*2
s = torch.Tensor([a['scale_provided_det']]) * scale_mul_factor
c = torch.Tensor(a['objpos_det'])
# For single-person pose estimation with a centered/scaled figure
# the image in the original size
img = imutils.load_image(img_path)
r = 0
s_rand = 1
if self.is_train: #data augmentation for training data
s_rand = (1 + sample_from_bounded_gaussian(self.scale_factor / 2.))
s = s * s_rand
r = sample_from_bounded_gaussian(self.rot_factor / 2.)
#print('s shape is ', s.size(), 's is ', s)
#if np.random.uniform(0, 1, 1) <= 0.6:
# r = np.array([0])
if self.use_flipping:
# Flip
if np.random.random() <= 0.5:
img = torch.from_numpy(HumanAug.fliplr(
img.numpy())).float()
pts = HumanAug.shufflelr(pts,
width=img.size(2),
dataset='face')
c[0] = img.size(2) - c[0]
# Color
img[0, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
img[1, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
img[2, :, :].mul_(np.random.uniform(0.6, 1.4)).clamp_(0, 1)
if self.use_occlusion:
# Apply a random black occlusion
# C x H x W
patch_center_row = randint(1, img.size(1))
patch_center_col = randint(1, img.size(2))
patch_height = randint(1, img.size(1) / 2)
patch_width = randint(1, img.size(2) / 2)
row_min = max(0, patch_center_row - patch_height)
row_max = min(img.size(1), patch_center_row + patch_height)
col_min = max(0, patch_center_col - patch_width)
col_max = min(img.size(2), patch_center_col + patch_width)
img[:, row_min:row_max, col_min:col_max] = 0
# Prepare points first
pts_input_res = HumanAug.TransformPts(pts.numpy(), c.numpy(),
s.numpy(), r, self.inp_res,
self.std_size)
# Some landmark points can go outside after transformation. Determine the
# extra scaling required.
# This can only be done for the training points. For validation, we do
# not know the points location.
if self.is_train and self.keep_pts_inside:
# visible copy takes care of whether point is visible or not.
visible_copy = visible_multiclass.copy()
visible_copy[visible_multiclass > 1] = 1
scale_down = get_ideal_scale(pts_input_res,
self.inp_res,
img_path,
visible=visible_copy)
s = s / scale_down
s_rand = s_rand / scale_down
pts_input_res = HumanAug.TransformPts(pts.numpy(), c.numpy(),
s.numpy(), r, self.inp_res,
self.std_size)
if a['dataset'] == "aflw":
meta_box_size = a['box_size']
# We convert the meta_box size also to the input res. The meta_box
# is not formed by the landmark point but is supplied externally.
# We assume the meta_box as two points [meta_box_size, 0] and [0, 0]
# apply the transformation on top of it
temp = HumanAug.TransformPts(
np.array([[meta_box_size, 0], [0, 0]]), c.numpy(), s.numpy(),
r, self.inp_res, self.std_size)
# Passed as array of 2 x 2
# we only want the transformed distance between the points
meta_box_size_input_res = np.linalg.norm(temp[1] - temp[0])
else:
meta_box_size_input_res = -10 # some invalid number
# pts_input_res is in the size of 256 x 256
# Bring down to 64 x 64 since finally heatmap will be 64 x 64
pts_aug = pts_input_res * (1. * self.out_res / self.inp_res)
# Prepare image
inp = HumanAug.crop(imutils.im_to_numpy(img), c.numpy(), s.numpy(), r,
self.inp_res, self.std_size)
inp_vis = inp
inp = imutils.im_to_torch(inp).float() # 3*256*256
# Generate proxy ground truth heatmap
heatmap, pts_aug = HumanPts.pts2heatmap(pts_aug,
[self.out_res, self.out_res],
sigma=self.sigma)
heatmap = torch.from_numpy(heatmap).float()
heatmap_mask = HumanPts.pts2mask(pts_aug, [self.out_res, self.out_res],
bb=10)
if self.is_train:
return inp, heatmap, pts_input_res, heatmap_mask, s_rand, visible_multiclass, meta_box_size_input_res
else:
return inp, heatmap, pts_input_res, c, s, index, inp_vis, s_rand, visible_multiclass, meta_box_size_input_res
shuffle=False,
num_workers=1,
pin_memory=True)
# Visualize some images
for i, (img, _, points, _, s) in enumerate(val_loader):
print(i)
image = img[index].numpy()
pts = points[index].numpy()
plt.figure(figsize=(16, 8))
plt.subplot(121)
plt.imshow(swap_channels(image))
plt_pts(plt, pts)
image = torch.from_numpy(HumanAug.fliplr(img[index].numpy())).float()
pts = HumanAug.shufflelr(points[index],
width=image.size(2),
dataset='face')
plt.subplot(122)
plt.imshow(swap_channels(image))
plt_pts(plt, pts)
plt.show()
plt.close()
if i > 10:
break
val_loader = torch.utils.data.DataLoader(FACE(
"dataset/all_300Wtest_train.json", ".", is_train=True),
batch_size=10,
def generateSampleFace(self, idx):
sf = self.scale_factor
rf = self.rot_factor
#print('Filename -->{}'.format(self.anno[idx][:-4] + '.jpg'))
main_pts = sio.loadmat(self.anno[idx])
pts = main_pts['pt3d_68'][0:2, :].transpose()
#print(pts.dtype)
pts = np.float32(pts)
pts = torch.from_numpy(pts)
#print('pts -> {}'.format(pts))
pts = torch.clamp(pts, min=0)
mins_ = torch.min(pts, 0)[0].view(2) # min vals
maxs_ = torch.max(pts, 0)[0].view(2) # max vals
c = torch.FloatTensor((maxs_[0] - (maxs_[0] - mins_[0]) / 2,
maxs_[1] - (maxs_[1] - mins_[1]) / 2))
#print('min Values format -> {}'.format(mins_.dtype))
#print('max Values format -> {}'.format(maxs_.dtype))
c[1] -= ((maxs_[1] - mins_[1]) * 0.12)
s = (maxs_[0] - mins_[0] + maxs_[1] - mins_[1]) / 195
img = load_image(self.anno[idx][:-4] + '.jpg')
r = 0
if self.is_train:
s = s * torch.randn(1).mul_(sf).add_(1).clamp(1 - sf, 1 + sf)[0]
r = torch.randn(1).mul_(rf).clamp(
-2 * rf, 2 * rf)[0] if random.random() <= 0.6 else 0
if random.random() <= 0.5:
img = torch.from_numpy(fliplr(img.numpy())).float()
pts = shufflelr(pts, width=img.size(2), dataset='aflw2000')
c[0] = img.size(2) - c[0]
img[0, :, :].mul_(random.uniform(0.7, 1.3)).clamp_(0, 1)
img[1, :, :].mul_(random.uniform(0.7, 1.3)).clamp_(0, 1)
img[2, :, :].mul_(random.uniform(0.7, 1.3)).clamp_(0, 1)
# Prepare image and groundtruth map
inp = HumanAug.crop(imutils.im_to_numpy(img), c.numpy(), s, r, 256,
200)
inp = imutils.im_to_torch(inp).float()
pts_input_res = HumanAug.TransformPts(pts.numpy(), c.numpy(), s, r,
256, 200)
pts_aug = pts_input_res * (1. * 64 / 256)
# Generate ground truth
heatmap, pts_aug = HumanPts.pts2heatmap(pts_aug, [64, 64], sigma=1)
heatmap = torch.from_numpy(heatmap).float()
# inp = crop(img, c, s, [256, 256], rot=r)
# # inp = color_normalize(inp, self.mean, self.std)
# tpts = pts.clone()
# out = torch.zeros(self.nParts, 64, 64)
# for i in range(self.nParts):
# if tpts[i, 0] > 0:
# tpts[i, 0:2] = to_torch(transform(tpts[i, 0:2] + 1, c, s, [64, 64], rot=r))
# out[i] = draw_labelmap(out[i], tpts[i] - 1, sigma=1)
return inp, heatmap, pts, c, s, pts_input_res
