def __getitem__(self, index):
assert index >= 0 and index < self.length, 'Invalid index : {:}'.format(
index)
images, is_video_or_not = parse_video_by_indicator(
self.datas[index], self.video_parser, False)
images = [pil_loader(image) for image in images]
target = self.labels[index].copy()
# transform the image and points
if self.transform is not None:
images, target = self.transform(images, target)
# obtain the visiable indicator vector
if target.is_none(): nopoints = True
else: nopoints = False
# If for evaluation not load label, keeps the original data
temp_save_wh = target.temp_save_wh
ori_size = torch.IntTensor([
temp_save_wh[1], temp_save_wh[0], temp_save_wh[2], temp_save_wh[3]
]) # H, W, Cropped_[x1,y1]
if isinstance(images[0], Image.Image):
height, width = images[0].size[1], images[0].size[0]
elif isinstance(images[0], torch.FloatTensor):
height, width = images[0].size(1), images[0].size(2)
else:
raise Exception('Unknown type of image : {}'.format(type(
images[0])))
if target.is_none() == False:
target.apply_bound(width, height)
points = target.points.copy()
points = torch.from_numpy(points.transpose(
(1, 0))).type(torch.FloatTensor)
Hpoint = target.points.copy()
else:
points = torch.from_numpy(np.zeros(
(self.NUM_PTS, 3))).type(torch.FloatTensor)
Hpoint = np.zeros((3, self.NUM_PTS))
heatmaps, mask = generate_label_map(Hpoint, height // self.downsample,
width // self.downsample,
self.sigma, self.downsample,
nopoints,
self.heatmap_type) # H*W*C
heatmaps = torch.from_numpy(heatmaps.transpose(
(2, 0, 1))).type(torch.FloatTensor)
mask = torch.from_numpy(mask.transpose(
(2, 0, 1))).type(torch.ByteTensor)
torch_index = torch.IntTensor([index])
torch_nopoints = torch.ByteTensor([nopoints])
video_indicator = torch.ByteTensor([is_video_or_not])
return torch.stack(
images
), heatmaps, mask, points, torch_index, torch_nopoints, video_indicator, ori_size
def __process_affine(self, image, target, theta, nopoints, aux_info=None):
image, target, theta = image.clone(), target.copy(), theta.clone()
(C, H, W), (height, width) = image.size(), self.shape
if nopoints: # do not have label
norm_trans_points = torch.zeros((3, self.NUM_PTS))
heatmaps = torch.zeros(
(self.NUM_PTS + 1, height // self.downsample,
width // self.downsample))
mask = torch.ones((self.NUM_PTS + 1, 1, 1), dtype=torch.uint8)
else:
norm_trans_points = apply_affine2point(target.get_points(), theta,
(H, W))
norm_trans_points = apply_boundary(norm_trans_points)
real_trans_points = norm_trans_points.clone()
real_trans_points[:2, :] = denormalize_points(
self.shape, real_trans_points[:2, :])
heatmaps, mask = generate_label_map(real_trans_points.numpy(),
height // self.downsample,
width // self.downsample,
self.sigma, self.downsample,
nopoints,
self.heatmap_type) # H*W*C
heatmaps = torch.from_numpy(heatmaps.transpose(
(2, 0, 1))).type(torch.FloatTensor)
mask = torch.from_numpy(mask.transpose(
(2, 0, 1))).type(torch.ByteTensor)
affineImage = affine2image(image, theta, self.shape)
return affineImage, heatmaps, mask, norm_trans_points, theta
def __process_affine(self, image, target, theta, nopoints, aux_info=None):
image, target, theta = image.clone(), target.copy(), theta.clone()
(C, H, W), (height, width) = image.size(), self.shape
if nopoints: # do not have label
norm_trans_points = torch.zeros((3, self.NUM_PTS))
heatmaps = torch.zeros((self.NUM_PTS + 1, height // self.downsample, width // self.downsample))
mask = torch.ones((self.NUM_PTS + 1, 1, 1), dtype=torch.uint8)
transpose_theta = identity2affine(False)
else:
norm_trans_points = apply_affine2point(target.get_points(), theta, (H, W))
norm_trans_points = apply_boundary(norm_trans_points)
real_trans_points = norm_trans_points.clone()
real_trans_points[:2, :] = denormalize_points(self.shape, real_trans_points[:2, :])
heatmaps, mask = generate_label_map(real_trans_points.numpy(), height // self.downsample,
width // self.downsample, self.sigma, self.downsample, nopoints,
self.heatmap_type) # H*W*C
heatmaps = torch.from_numpy(heatmaps.transpose((2, 0, 1))).type(torch.FloatTensor)
mask = torch.from_numpy(mask.transpose((2, 0, 1))).type(torch.ByteTensor)
if self.mean_face is None:
# warnings.warn('In LandmarkDataset use identity2affine for transpose_theta because self.mean_face is None.')
transpose_theta = identity2affine(False)
else:
if torch.sum(norm_trans_points[2, :] == 1) < 3:
warnings.warn(
'In LandmarkDataset after transformation, no visiable point, using identity instead. Aux: {:}'.format(
aux_info))
transpose_theta = identity2affine(False)
else:
transpose_theta = solve2theta(norm_trans_points, self.mean_face.clone())
affineImage = affine2image(image, theta, self.shape)
if self.cutout is not None: affineImage = self.cutout(affineImage)
return affineImage, heatmaps, mask, norm_trans_points, theta, transpose_theta
def _process_(self, image, target, index):
# transform the image and points
if self.transform is not None:
image, target = self.transform(image, target)
# obtain the visiable indicator vector
if target.is_none(): nopoints = True
else: nopoints = False
# If for evaluation not load label, keeps the original data
temp_save_wh = target.temp_save_wh
ori_size = torch.IntTensor([
temp_save_wh[1], temp_save_wh[0], temp_save_wh[2], temp_save_wh[3]
]) # H, W, Cropped_[x1,y1]
if isinstance(image, Image.Image):
height, width = image.size[1], image.size[0]
elif isinstance(image, torch.FloatTensor):
height, width = image.size(1), image.size(2)
else:
raise Exception('Unknown type of image : {}'.format(type(image)))
if target.is_none() == False:
target.apply_bound(width, height)
points = target.points.copy()
points = torch.from_numpy(points.transpose(
(1, 0))).type(torch.FloatTensor)
Hpoint = target.points.copy()
else:
points = torch.from_numpy(np.zeros(
(self.NUM_PTS, 3))).type(torch.FloatTensor)
Hpoint = np.zeros((3, self.NUM_PTS))
heatmaps, mask = generate_label_map(Hpoint, height // self.downsample,
width // self.downsample,
self.sigma, self.downsample,
nopoints,
self.heatmap_type) # H*W*C
heatmaps = torch.from_numpy(heatmaps.transpose(
(2, 0, 1))).type(torch.FloatTensor)
mask = torch.from_numpy(mask.transpose(
(2, 0, 1))).type(torch.BoolTensor)
torch_index = torch.IntTensor([index])
torch_nopoints = torch.BoolTensor([nopoints])
return image, heatmaps, mask, points, torch_index, torch_nopoints, ori_size
def __getitem__(self, index):
assert index >= 0 and index < self.length, 'Invalid index : {:}'.format(index)
images, is_video_or_not = parse_video_by_indicator(self.datas[index], self.video_parser, False)
images = [pil_loader(image) for image in images]
target = self.labels[index].copy()
# transform the image and points
if self.transform is not None:
images, target = self.transform(images, target)
# obtain the visiable indicator vector
if target.is_none(): nopoints = True
else : nopoints = False
# If for evaluation not load label, keeps the original data
temp_save_wh = target.temp_save_wh
ori_size = torch.IntTensor( [temp_save_wh[1], temp_save_wh[0], temp_save_wh[2], temp_save_wh[3]] ) # H, W, Cropped_[x1,y1]
if isinstance(images[0], Image.Image):
height, width = images[0].size[1], images[0].size[0]
elif isinstance(images[0], torch.FloatTensor):
height, width = images[0].size(1), images[0].size(2)
else:
raise Exception('Unknown type of image : {}'.format( type(images[0]) ))
if target.is_none() == False:
target.apply_bound(width, height)
points = target.points.copy()
points = torch.from_numpy(points.transpose((1,0))).type(torch.FloatTensor)
Hpoint = target.points.copy()
else:
points = torch.from_numpy(np.zeros((self.NUM_PTS,3))).type(torch.FloatTensor)
Hpoint = np.zeros((3, self.NUM_PTS))
heatmaps, mask = generate_label_map(Hpoint, height//self.downsample, width//self.downsample, self.sigma, self.downsample, nopoints, self.heatmap_type) # H*W*C
heatmaps = torch.from_numpy(heatmaps.transpose((2, 0, 1))).type(torch.FloatTensor)
mask = torch.from_numpy(mask.transpose((2, 0, 1))).type(torch.ByteTensor)
torch_index = torch.IntTensor([index])
torch_nopoints = torch.ByteTensor( [ nopoints ] )
video_indicator = torch.ByteTensor( [is_video_or_not] )
return torch.stack(images), heatmaps, mask, points, torch_index, torch_nopoints, video_indicator, ori_size
def _process_(self, image, target, index):
# transform the image and points
if self.transform is not None:
image, target = self.transform(image, target)
# obtain the visiable indicator vector
if target.is_none(): nopoints = True
else : nopoints = False
# If for evaluation not load label, keeps the original data
temp_save_wh = target.temp_save_wh
ori_size = torch.IntTensor( [temp_save_wh[1], temp_save_wh[0], temp_save_wh[2], temp_save_wh[3]] ) # H, W, Cropped_[x1,y1]
if isinstance(image, Image.Image):
height, width = image.size[1], image.size[0]
elif isinstance(image, torch.FloatTensor):
height, width = image.size(1), image.size(2)
else:
raise Exception('Unknown type of image : {}'.format( type(image) ))
if target.is_none() == False:
target.apply_bound(width, height)
points = target.points.copy()
points = torch.from_numpy(points.transpose((1,0))).type(torch.FloatTensor)
Hpoint = target.points.copy()
else:
points = torch.from_numpy(np.zeros((self.NUM_PTS,3))).type(torch.FloatTensor)
Hpoint = np.zeros((3, self.NUM_PTS))
heatmaps, mask = generate_label_map(Hpoint, height//self.downsample, width//self.downsample, self.sigma, self.downsample, nopoints, self.heatmap_type) # H*W*C
heatmaps = torch.from_numpy(heatmaps.transpose((2, 0, 1))).type(torch.FloatTensor)
mask = torch.from_numpy(mask.transpose((2, 0, 1))).type(torch.ByteTensor)
torch_index = torch.IntTensor([index])
torch_nopoints = torch.ByteTensor( [ nopoints ] )
return image, heatmaps, mask, points, torch_index, torch_nopoints, ori_size
def __process_affine(self,
frames,
target,
theta,
nopoints,
skipopt,
aux_info=None):
frames, target, theta = [frame.clone() for frame in frames
], target.copy(), theta.clone()
(C, H, W), (height, width) = frames[0].size(), self.shape
if nopoints: # do not have label
norm_trans_points = torch.zeros((3, self.NUM_PTS))
heatmaps = torch.zeros(
(self.NUM_PTS + 1, height // self.downsample,
width // self.downsample))
mask = torch.ones((self.NUM_PTS + 1, 1, 1), dtype=torch.uint8)
else:
norm_trans_points = apply_affine2point(target.get_points(), theta,
(H, W))
norm_trans_points = apply_boundary(norm_trans_points)
real_trans_points = norm_trans_points.clone()
real_trans_points[:2, :] = denormalize_points(
self.shape, real_trans_points[:2, :])
heatmaps, mask = generate_label_map(real_trans_points.numpy(),
height // self.downsample,
width // self.downsample,
self.sigma, self.downsample,
nopoints,
self.heatmap_type) # H*W*C
heatmaps = torch.from_numpy(heatmaps.transpose(
(2, 0, 1))).type(torch.FloatTensor)
mask = torch.from_numpy(mask.transpose(
(2, 0, 1))).type(torch.ByteTensor)
affineFrames = [
affine2image(frame, theta, self.shape) for frame in frames
]
if not skipopt:
Gframes = [self.tensor2img(frame) for frame in affineFrames]
forward_flow, backward_flow = [], []
for idx in range(len(Gframes)):
if idx > 0:
forward_flow.append(
self.optflow.calc(Gframes[idx - 1], Gframes[idx],
None))
if idx + 1 < len(Gframes):
#backward_flow.append( self.optflow.calc(Gframes[idx], Gframes[idx+1], None) ) ## HDXY
backward_flow.append(
self.optflow.calc(Gframes[idx + 1], Gframes[idx],
None))
forward_flow = torch.stack(
[torch.from_numpy(x) for x in forward_flow])
backward_flow = torch.stack(
[torch.from_numpy(x) for x in backward_flow])
else:
forward_flow, backward_flow = torch.zeros(
(len(affineFrames) - 1, height, width, 2)), torch.zeros(
(len(affineFrames) - 1, height, width, 2))
# affineFrames #frames x #channel x #height x #width
# forward_flow (#frames-1) x #height x #width x 2
# backward_flow (#frames-1) x #height x #width x 2
return torch.stack(
affineFrames
), forward_flow, backward_flow, heatmaps, mask, norm_trans_points, theta
def __process_affine(self,
frames,
target,
theta,
nopoints,
skip_opt,
aux_info=None):
frames, target, theta = [frame.clone() for frame in frames
], target.copy(), theta.clone()
(C, H, W), (height, width) = frames[0].size(), self.shape
if nopoints: # do not have label
norm_trans_points = torch.zeros((3, self.NUM_PTS))
heatmaps = torch.zeros(
(self.NUM_PTS + 1, height // self.downsample,
width // self.downsample))
mask = torch.ones((self.NUM_PTS + 1, 1, 1), dtype=torch.uint8)
transpose_theta = identity2affine(False)
else:
norm_trans_points = apply_affine2point(target.get_points(), theta,
(H, W))
norm_trans_points = apply_boundary(norm_trans_points)
real_trans_points = norm_trans_points.clone()
real_trans_points[:2, :] = denormalize_points(
self.shape, real_trans_points[:2, :])
heatmaps, mask = generate_label_map(real_trans_points.numpy(),
height // self.downsample,
width // self.downsample,
self.sigma, self.downsample,
nopoints,
self.heatmap_type) # H*W*C
heatmaps = torch.from_numpy(heatmaps.transpose(
(2, 0, 1))).type(torch.FloatTensor)
mask = torch.from_numpy(mask.transpose(
(2, 0, 1))).type(torch.ByteTensor)
if torch.sum(norm_trans_points[2, :] ==
1) < 3 or self.mean_face is None:
warnings.warn(
'In GeneralDatasetV2 after transformation, no visiable point, using identity instead. Aux: {:}'
.format(aux_info))
transpose_theta = identity2affine(False)
else:
transpose_theta = solve2theta(norm_trans_points,
self.mean_face.clone())
affineFrames = [
affine2image(frame, theta, self.shape) for frame in frames
]
if not skip_opt:
Gframes = [self.tensor2img(frame) for frame in affineFrames]
forward_flow, backward_flow = [], []
for idx in range(len(Gframes)):
if idx > 0:
forward_flow.append(
self.optflow.calc(Gframes[idx - 1], Gframes[idx],
None))
if idx + 1 < len(Gframes):
#backward_flow.append( self.optflow.calc(Gframes[idx], Gframes[idx+1], None) )
backward_flow.append(
self.optflow.calc(Gframes[idx + 1], Gframes[idx],
None))
forward_flow = torch.stack(
[torch.from_numpy(x) for x in forward_flow])
backward_flow = torch.stack(
[torch.from_numpy(x) for x in backward_flow])
else:
forward_flow, backward_flow = torch.zeros(
(len(affineFrames) - 1, height, width, 2)), torch.zeros(
(len(affineFrames) - 1, height, width, 2))
# affineFrames #frames x #channel x #height x #width
# forward_flow (#frames-1) x #height x #width x 2
# backward_flow (#frames-1) x #height x #width x 2
return torch.stack(
affineFrames
), forward_flow, backward_flow, heatmaps, mask, norm_trans_points, theta, transpose_theta