def __getitem__(self, idx):
name = self.db[idx]
label = self.anno[name]
image_path = os.path.join(self.cfg.ROOT, name + '.png')
img = load_image(image_path, mode='RGB') # already / 255
coor = label['uv_coor']
# coor[1:,:] = coor[1:,:].reshape(5,4,-1)[:,::-1,:].reshape(20, -1)
coor = np.array(coor)
coor = to_torch(coor)
#apply transforms into image and calculate cooresponding coor
# if self.cfg.TRANSFORMS:
# img, coor = self.transforms(self.cfg.TRANSFORMS, img , coor)
meta = edict({'name': name})
heatmap = torch.zeros(22, img.size(1), img.size(2))
for i in range(21):
heatmap[i] = draw_heatmap(heatmap[i], coor[i],
self.cfg.HEATMAP.SIGMA)
return {
'input': {
'img': img
},
'heatmap': heatmap,
'coor': to_torch(coor[:, :2]),
'weight': 1,
'meta': meta
}
def __getitem__(self, idx):
name = self.db[idx]
label = self.anno[name]
image_path = os.path.join(self.cfg.ROOT, 'color', name + '.png')
img = load_image(image_path) #C * W * H
coor = to_torch(label['xyz'])
project_coor = label['project']
# print(project_coor[:, 0].min(), project_coor[:, 1].max())
assert project_coor[:, :2].min() >= 0 and project_coor[:, :2].max(
) < 320
matrix = label['K']
# norm the pose
# index_bone_length = torch.norm(coor[12,:] - coor[11,:])
# coor[0, :] = (coor[0] + coor[12]) / 2.
# coor = coor - coor[:1,:].repeat(21,1)
#apply transforms into image and calculate cooresponding coor and camera instrict matrix
if self.cfg.TRANSFORMS:
img, coor, project_coor, matrix = self.transforms(
self.cfg.TRANSFORMS, img, coor, project_coor, matrix)
# print(project_coor[:, 0].max())
assert project_coor[:, :2].min() >= 0 and project_coor[:, :2].max(
) < 256
matrix = np.linalg.inv(matrix) #take the inversion of matrix
meta = edict({'name': name})
isleft = name[-1] == 'L'
#corresponding depth position in depth map
project_coor = torch.tensor(project_coor).long()
index = torch.tensor([
i * img.size(1) * img.size(2) + project_coor[i, 0] * img.size(1) +
project_coor[i, 1] for i in range(21)
])
assert index.max() < img.size(1) * img.size(2) * 21, 'Wrong Position'
heatmap = torch.zeros(self.cfg.NUM_JOINTS, img.size(1), img.size(2))
for i in range(21):
heatmap[i] = draw_heatmap(heatmap[i], project_coor[i],
self.cfg.HEATMAP.SIGMA)
return {
'input': {
'img': img,
'hand_side': torch.tensor([isleft, 1 - isleft]).float(),
},
'index': index,
'matrix': to_torch(matrix),
# 'index_bone_length': index_bone_length,
'heatmap': heatmap,
'coor': to_torch(coor),
'project': to_torch(project_coor),
'weight': 1,
'meta': meta
}
def __getitem__(self, idx):
name = self.db[idx]
label = self.anno[name]
image_path = os.path.join(self.cfg.ROOT, name + '.png')
img = load_image(image_path, mode='RGB') # already / 255
coor = label['uv_coor']
coor = np.array(coor)
coor = to_torch(coor)
meta = edict({'name': name})
return {'img': img, 'coor': to_torch(coor[:, :2]), 'meta': meta}
def __getitem__(self, idx):
name = self.db[idx]
label = self.anno[name]
image_path = os.path.join(self.cfg.ROOT, 'color', name + '.png')
img = load_image(image_path) # already / 255 with C * W * H
depth_path = os.path.join(self.cfg.ROOT, 'depth', name + '.torch')
depthmap = torch.load(depth_path).unsqueeze(0)
coor2d = label['project']
matrix = label['K']
assert coor2d[:, :2].min() >= 0 and coor2d[:, :2].max() < 320
meta = edict({'name': name})
coor2d[1:, :] = coor2d[1:, :].reshape(5, 4,
-1)[:, ::-1, :].reshape(20, -1)
coor2d = to_torch(np.array(coor2d))
coor2d[:, :2] = coor2d[:, :2].long().float()
return {
"img": img,
"matrix": matrix,
"coor2d": coor2d,
"depthmap": depthmap,
"meta": meta
}
def __getitem__(self, idx):
w = self.db[idx]
image_path = os.path.join(self.cfg.ROOT, w[1], w[1] + w[2], 'image',
w[0] + '.png')
label_path = os.path.join(self.cfg.ROOT, w[1], w[1] + w[2], 'label',
w[0] + '.json')
img = load_image(image_path, mode='GBR') # C * H * W
label = json.load(open(label_path))
#calculate ground truth coordination
coor = np.array(label['perspective'])
coor[:, 0] = coor[:, 0] * img.size(1)
coor[:, 1] = (1 - coor[:, 1]) * img.size(2)
coor = coor[:, :2]
#apply transforms into image and calculate cooresponding coor
if self.cfg.TRANSFORMS:
img, coor = self.transforms(self.cfg.TRANSFORMS, img, coor)
#openpose require 22 channel, discard the last one
# heatmap = np.zeros((self.cfg.NUM_JOINTS, img.shape[1], img.shape[2]))
# for i in range(self.cfg.NUM_JOINTS - 1):
# heatmap[i, :, :] = draw_heatmap(heatmap[i], coor[i], self.cfg.HEATMAP.SIGMA, type = self.cfg.HEATMAP.TYPE)
meta = edict({'name': w[1] + ' ' + w[2] + ' ' + w[0]})
assert coor.min() > 0, label_path
return {'img': img, 'coor': to_torch(coor), 'meta': meta}
def __getitem__(self, idx):
name = self.names[idx // 5220]
path = self.db[name][idx % 5220]
coor = self.anno[name][idx % 5220]
img = load_image(path)
#calculate ground truth coordination
coor = torch.tensor(coor)
coor[:, 0] = coor[:, 0] * img.size(1)
coor[:, 1] = (1 - coor[:, 1]) * img.size(2)
coor = coor[:, :2]
#apply transforms into image and calculate cooresponding coor
# if self.cfg.TRANSFORMS:
# img, coor = self.transforms(self.cfg.TRANSFORMS, img , coor)
#openpose require 22 channel, discard the last one
# heatmap = np.zeros((self.cfg.NUM_JOINTS, img.shape[1], img.shape[2]))
# for i in range(self.cfg.NUM_JOINTS - 1):
# heatmap[i, :, :] = draw_heatmap(heatmap[i], coor[i], self.cfg.HEATMAP.SIGMA, type = self.cfg.HEATMAP.TYPE)
meta = edict({'name': path})
assert coor.min() > 0, path
return {'img': img, 'coor': to_torch(coor), 'meta': meta}
def __getitem__(self, idx):
name = self.names[idx // 5220]
path = self.db[name][idx % 5220]
coor = self.anno[name][idx % 5220]
img = load_image(path)
#calculate ground truth coordination
coor = torch.tensor(coor)
coor[:, 0] = coor[:, 0] * img.size(1)
coor[:, 1] = (1 - coor[:, 1]) * img.size(2)
coor = coor[:, :2]
#apply transforms into image and calculate cooresponding coor
if self.cfg.TRANSFORMS:
img, coor = self.transforms(self.cfg.TRANSFORMS, img, coor)
#openpose require 22 channel, discard the last one
heatmap = np.zeros((self.cfg.NUM_JOINTS, img.shape[1], img.shape[2]))
for i in range(self.cfg.NUM_JOINTS - 1):
heatmap[i, :, :] = draw_heatmap(heatmap[i],
coor[i],
self.cfg.HEATMAP.SIGMA,
type=self.cfg.HEATMAP.TYPE)
# print (name, coor)
# fig = plt.figure(1)
# ax = fig.add_subplot(111)
# plot_hand(im_to_numpy(img), coor, ax)
# plt.show()
meta = edict({'name': path})
assert coor.min() > 0, path
return {
'input': {
'img': img
},
'coor': to_torch(coor),
'heatmap': to_torch(heatmap),
'weight': 1,
'meta': meta
}
def reprocess(self, input, reprocess_cfg):
img = input['img']
coor2d = input['coor2d']
matrix = input['matrix']
meta = input['meta']
#apply transforms into image and calculate cooresponding coor and camera instrict matrix
if reprocess_cfg.TRANSFORMS:
img, _, coor2d, matrix = transforms(reprocess_cfg.TRANSFORMS, img,
None, coor2d, matrix)
matrix = np.linalg.inv(matrix) #take the inversion of matrix
coor3d = coor2d.clone()
coor3d[:, :2] *= coor3d[:, 2:]
coor3d = torch.matmul(coor3d, to_torch(matrix).transpose(0, 1))
root_depth = coor2d[0, 2].clone()
index_bone_length = torch.norm(coor3d[9, :] - coor3d[10, :])
relative_depth = (coor2d[:, 2] - root_depth) / index_bone_length
heatmap = torch.zeros(reprocess_cfg.NUM_JOINTS, img.size(1),
img.size(2))
for i in range(21):
heatmap[i] = draw_heatmap(heatmap[i], coor2d[i],
reprocess_cfg.HEATMAP.SIGMA)
return {
'input': {
'img': img
},
'heatmap': heatmap,
'matrix': to_torch(matrix),
'color_hm': heatmap,
'coor3d': to_torch(coor3d),
'coor2d': to_torch(coor2d),
'root_depth': root_depth,
'index_bone_length': index_bone_length,
'relative_depth': relative_depth,
'weight': 1,
'meta': meta,
}
def __getitem__(self, idx):
w = self.db[idx]
image_path = os.path.join(self.cfg.ROOT, w[1], w[1] + w[2], 'image',
w[0] + '.png')
label_path = os.path.join(self.cfg.ROOT, w[1], w[1] + w[2], 'label',
w[0] + '.json')
img = load_image(image_path)
label = json.load(open(label_path))
#calculate ground truth coordination
coor = torch.tensor(label['camera']).numpy()
coor[1:, :] = coor[1:, :].reshape(5, 4, -1)[:, ::-1, :].reshape(
20, -1) #iccv order !
coor = np.array(coor)
coor = to_torch(coor)
coor = coor - coor[:1, :].repeat(21, 1)
index_bone_length = torch.norm(coor[12, :] - coor[11, :])
#apply transforms into image and calculate cooresponding coor
if self.cfg.TRANSFORMS:
img, label = self.transforms(self.cfg.TRANSFORMS, img, coor)
meta = edict({'name': w[1] + ' ' + w[2] + ' ' + w[0]})
return {
'input': {
'img': img,
'hand_side': torch.tensor([0, 1]).float()
},
'coor': to_torch(coor),
# 'heat_map': to_torch(heat_map),
'index_bone_length': index_bone_length,
'weight': 1,
'meta': meta
}
def __getitem__(self, idx):
name = self.names[idx // 5220]
path = self.db[name][idx % 5220]
coor = self.anno[name][idx % 5220]
img = load_image(path)
#calculate ground truth coordination
coor = torch.tensor(coor).numpy()
coor[1:, :] = coor[1:, :].reshape(5, 4, -1)[:, ::-1, :].reshape(
20, -1) #iccv order !
coor = np.array(coor)
coor = to_torch(coor)
coor = coor - coor[:1, :].repeat(21, 1)
index_bone_length = torch.norm(coor[12, :] - coor[11, :])
#apply transforms into image and calculate cooresponding coor
if self.cfg.TRANSFORMS:
img, label = self.transforms(self.cfg.TRANSFORMS, img, coor)
# heat_map = np.zeros((self.cfg.NUM_JOINTS, img.shape[1], img.shape[2]))
# for i in range(self.cfg.NUM_JOINTS):
# heat_map[i, :, :] = draw_heatmap(heat_map[i], coor[i], self.cfg.HEATMAP.SIGMA, type = self.cfg.HEATMAP.TYPE)
meta = edict({'name': path})
return {
'input': {
'img': img,
'hand_side': torch.tensor([0, 1]).float()
},
'coor': to_torch(coor),
# 'heat_map': to_torch(heat_map),
'index_bone_length': index_bone_length,
'weight': 1,
'meta': meta
}
def __getitem__(self, idx):
name = self.name[idx // 1500]
coor2d = self.db[name]['sk']['coor2d'][idx % 1500,:,:]
matrix = self.db[name]['sk']['matrix'][idx % 1500]
name = name.split('_')
image_path = os.path.join(self.cfg.ROOT, name[0], 'SK_' + str(idx % 1500) + '.png')
img = load_image(image_path, mode = 'RGB')
depth_path = os.path.join(self.cfg.ROOT, name[0], 'SK_depth_{}.pickle'.format(idx % 1500))
depthmap = pickle.load(open(depth_path)).unsqueeze(0)
# coor2d = label['project']
# assert coor2d[:, :2].min() >= 0 and coor2d[:, :2].max() < 320
# coor2d = torch.matmul(coor3d, to_torch(matrix).transpose(0, 1))
meta = edict({'name': name})
return {'img':img,
'depthmap': depthmap,
# 'index': index,
'matrix': to_torch(matrix),
'coor2d': to_torch(coor2d),
'meta': meta,
}
def __getitem__(self, idx):
name = self.name[idx // 1500]
coor = self.db[idx // 1500][:, :, idx % 1500]
coor = coor.transpose(1, 0) / 1000. #milionmeter to meter
coor[1:, ] = coor[1:, :].reshape(5, 4,
-1)[::-1, ::-1, :].reshape(20, -1)
coor = np.array(coor)
coor = to_torch(coor)
index_bone_length = torch.norm(coor[12, :] - coor[11, :])
coor = coor - coor[:1, :].repeat(21, 1)
name = name.split("_")
if name[1] == 'BB':
image_path = os.path.join(
self.cfg.ROOT, name[0],
"_".join(['left', str(idx % 1500)]) + '.png')
elif name[1] == 'SK':
image_path = os.path.join(
self.cfg.ROOT, name[0],
"_".join(['SK', str(idx % 1500)]) + '.png')
else:
raise Exception("Unrecognized name {}".format(name))
img = load_image(image_path)
#apply transforms into image and calculate cooresponding coor
if self.cfg.TRANSFORMS:
img, coor = self.transforms(self.cfg.TRANSFORMS, img, coor)
meta = edict({'name': name})
isleft = 1
return {
'input': {
'img': img,
'hand_side': torch.tensor([isleft, 1 - isleft]).float(),
# 'heatmap': heatmap
},
"index_bone_length": index_bone_length,
'coor': coor,
'weight': 1,
'meta': meta
}
def __getitem__(self, idx):
tok = self.db[idx].split('/')
image_path = os.path.join(self.cfg.ROOT, tok[0], 'image',
tok[1] + '.jpg')
img = load_image(image_path, mode='RGB')
label_path = os.path.join(self.cfg.ROOT, tok[0], 'annotation',
tok[1] + '.torch')
label = torch.load(label_path)
depth_path = 'data/RHD/cropped/training/depth/' + self.depth_db[
random.randint(0,
len(self.depth_db) - 1)]
depthmap = torch.load(depth_path).unsqueeze(0)
meta = edict({'name': self.db[idx]})
return {
'img': img,
'depthmap': depthmap,
'matrix': to_torch(label['matrix']).float(),
'coor2d': label['coor2d'].float(),
'meta': meta,
}
def __getitem__(self, idx):
name = self.name[idx // 1500]
coor = self.db[idx // 1500][idx % 1500, :, :]
coor = to_torch(coor)
name = name.split("_")
image_path = os.path.join(self.cfg.ROOT, name[0],
name[1] + '_' + str(idx % 1500) + '.png')
img = load_image(image_path, mode='RGB')
#apply transforms into image and calculate cooresponding coor
if self.cfg.TRANSFORMS:
img, coor = self.transforms(self.cfg.TRANSFORMS, img, coor)
# print (name, idx % 1500, coor)
# fig = plt.figure(1)
# ax = fig.add_subplot(111)
# plot_hand(im_to_numpy(img), coor, ax)
# plt.show()
meta = edict({'name': name})
heatmap = torch.zeros(self.cfg.NUM_JOINTS, img.size(1), img.size(2))
for i in range(self.cfg.NUM_JOINTS - 1):
heatmap[i] = draw_heatmap(heatmap[i], coor[i],
self.cfg.HEATMAP.SIGMA)
return {
'input': {
'img': img
},
'heatmap': heatmap,
'coor': coor,
'weight': 1,
'meta': meta
}
def reprocess(input, reprocess_cfg):
img = input['img']
depthmap = input['depthmap']
coor2d = input['coor2d']
# assert coor2d[:, :2].min() >= 0 and coor2d[:, :2].max() < 320
matrix = input['matrix']
meta = input['meta']
#apply transforms into image and calculate cooresponding coor and camera instrict matrix
if reprocess_cfg.TRANSFORMS:
img, depthmap, coor2d, matrix = transforms(reprocess_cfg.TRANSFORMS,
img, depthmap, coor2d,
matrix)
# if depthmap_max - depthmap_min < 1e-6:
# print(name, ": ", depthmap_max - depthmap_min)
# depthmap = (depthmap.max() - depthmap) / (depthmap_max - depthmap_min)
# print(depthmap)
matrix = np.linalg.inv(matrix) #take the inversion of matrix
coor3d = coor2d.clone()
coor3d[:, :2] *= coor3d[:, 2:]
coor3d = torch.matmul(coor3d, to_torch(matrix).transpose(0, 1))
root_depth = coor2d[0, 2].clone()
index_bone_length = torch.norm(coor3d[9, :] - coor3d[10, :])
relative_depth = (coor2d[:, 2] - root_depth) / index_bone_length
depthmap *= float(2**16 - 1)
depthmap = (depthmap - root_depth) / index_bone_length
depthmap_max = depthmap.max()
depthmap_min = depthmap.min()
depthmap = (depthmap - depthmap_min) / (depthmap_max - depthmap_min)
heatmap = torch.zeros(reprocess_cfg.NUM_JOINTS, img.size(1), img.size(2))
depth = torch.zeros(reprocess_cfg.NUM_JOINTS, img.size(1), img.size(2))
for i in range(21):
heatmap[i] = draw_heatmap(heatmap[i], coor2d[i],
reprocess_cfg.HEATMAP.SIGMA)
depth[i] = heatmap[i] * (coor2d[i, 2] -
coor2d[0, 2]) / index_bone_length
return {
'input': {
'img': img,
'depthmap': depthmap,
},
'heatmap': heatmap,
'matrix': to_torch(matrix),
'color_hm': heatmap,
'depth_hm': depth,
'depthmap': depthmap,
'depthmap_max': depthmap_max,
'depthmap_range': depthmap_max - depthmap_min,
'coor3d': to_torch(coor3d),
'coor2d': to_torch(coor2d),
'root_depth': root_depth,
'index_bone_length': index_bone_length,
'relative_depth': relative_depth,
'weight': 1,
'meta': meta,
}