def __getitem__(self, idx): name = self.db[idx] label = self.anno[name] image_path = os.path.join(self.cfg.ROOT, name + '.png') img = load_image(image_path, mode='RGB') # already / 255 coor = label['uv_coor'] # coor[1:,:] = coor[1:,:].reshape(5,4,-1)[:,::-1,:].reshape(20, -1) coor = np.array(coor) coor = to_torch(coor) #apply transforms into image and calculate cooresponding coor # if self.cfg.TRANSFORMS: # img, coor = self.transforms(self.cfg.TRANSFORMS, img , coor) meta = edict({'name': name}) heatmap = torch.zeros(22, img.size(1), img.size(2)) for i in range(21): heatmap[i] = draw_heatmap(heatmap[i], coor[i], self.cfg.HEATMAP.SIGMA) return { 'input': { 'img': img }, 'heatmap': heatmap, 'coor': to_torch(coor[:, :2]), 'weight': 1, 'meta': meta }
def __getitem__(self, idx): name = self.db[idx] label = self.anno[name] image_path = os.path.join(self.cfg.ROOT, 'color', name + '.png') img = load_image(image_path) #C * W * H coor = to_torch(label['xyz']) project_coor = label['project'] # print(project_coor[:, 0].min(), project_coor[:, 1].max()) assert project_coor[:, :2].min() >= 0 and project_coor[:, :2].max( ) < 320 matrix = label['K'] # norm the pose # index_bone_length = torch.norm(coor[12,:] - coor[11,:]) # coor[0, :] = (coor[0] + coor[12]) / 2. # coor = coor - coor[:1,:].repeat(21,1) #apply transforms into image and calculate cooresponding coor and camera instrict matrix if self.cfg.TRANSFORMS: img, coor, project_coor, matrix = self.transforms( self.cfg.TRANSFORMS, img, coor, project_coor, matrix) # print(project_coor[:, 0].max()) assert project_coor[:, :2].min() >= 0 and project_coor[:, :2].max( ) < 256 matrix = np.linalg.inv(matrix) #take the inversion of matrix meta = edict({'name': name}) isleft = name[-1] == 'L' #corresponding depth position in depth map project_coor = torch.tensor(project_coor).long() index = torch.tensor([ i * img.size(1) * img.size(2) + project_coor[i, 0] * img.size(1) + project_coor[i, 1] for i in range(21) ]) assert index.max() < img.size(1) * img.size(2) * 21, 'Wrong Position' heatmap = torch.zeros(self.cfg.NUM_JOINTS, img.size(1), img.size(2)) for i in range(21): heatmap[i] = draw_heatmap(heatmap[i], project_coor[i], self.cfg.HEATMAP.SIGMA) return { 'input': { 'img': img, 'hand_side': torch.tensor([isleft, 1 - isleft]).float(), }, 'index': index, 'matrix': to_torch(matrix), # 'index_bone_length': index_bone_length, 'heatmap': heatmap, 'coor': to_torch(coor), 'project': to_torch(project_coor), 'weight': 1, 'meta': meta }
def __getitem__(self, idx): name = self.db[idx] label = self.anno[name] image_path = os.path.join(self.cfg.ROOT, name + '.png') img = load_image(image_path, mode='RGB') # already / 255 coor = label['uv_coor'] coor = np.array(coor) coor = to_torch(coor) meta = edict({'name': name}) return {'img': img, 'coor': to_torch(coor[:, :2]), 'meta': meta}
def __getitem__(self, idx): name = self.db[idx] label = self.anno[name] image_path = os.path.join(self.cfg.ROOT, 'color', name + '.png') img = load_image(image_path) # already / 255 with C * W * H depth_path = os.path.join(self.cfg.ROOT, 'depth', name + '.torch') depthmap = torch.load(depth_path).unsqueeze(0) coor2d = label['project'] matrix = label['K'] assert coor2d[:, :2].min() >= 0 and coor2d[:, :2].max() < 320 meta = edict({'name': name}) coor2d[1:, :] = coor2d[1:, :].reshape(5, 4, -1)[:, ::-1, :].reshape(20, -1) coor2d = to_torch(np.array(coor2d)) coor2d[:, :2] = coor2d[:, :2].long().float() return { "img": img, "matrix": matrix, "coor2d": coor2d, "depthmap": depthmap, "meta": meta }
def __getitem__(self, idx): w = self.db[idx] image_path = os.path.join(self.cfg.ROOT, w[1], w[1] + w[2], 'image', w[0] + '.png') label_path = os.path.join(self.cfg.ROOT, w[1], w[1] + w[2], 'label', w[0] + '.json') img = load_image(image_path, mode='GBR') # C * H * W label = json.load(open(label_path)) #calculate ground truth coordination coor = np.array(label['perspective']) coor[:, 0] = coor[:, 0] * img.size(1) coor[:, 1] = (1 - coor[:, 1]) * img.size(2) coor = coor[:, :2] #apply transforms into image and calculate cooresponding coor if self.cfg.TRANSFORMS: img, coor = self.transforms(self.cfg.TRANSFORMS, img, coor) #openpose require 22 channel, discard the last one # heatmap = np.zeros((self.cfg.NUM_JOINTS, img.shape[1], img.shape[2])) # for i in range(self.cfg.NUM_JOINTS - 1): # heatmap[i, :, :] = draw_heatmap(heatmap[i], coor[i], self.cfg.HEATMAP.SIGMA, type = self.cfg.HEATMAP.TYPE) meta = edict({'name': w[1] + ' ' + w[2] + ' ' + w[0]}) assert coor.min() > 0, label_path return {'img': img, 'coor': to_torch(coor), 'meta': meta}
def __getitem__(self, idx): name = self.names[idx // 5220] path = self.db[name][idx % 5220] coor = self.anno[name][idx % 5220] img = load_image(path) #calculate ground truth coordination coor = torch.tensor(coor) coor[:, 0] = coor[:, 0] * img.size(1) coor[:, 1] = (1 - coor[:, 1]) * img.size(2) coor = coor[:, :2] #apply transforms into image and calculate cooresponding coor # if self.cfg.TRANSFORMS: # img, coor = self.transforms(self.cfg.TRANSFORMS, img , coor) #openpose require 22 channel, discard the last one # heatmap = np.zeros((self.cfg.NUM_JOINTS, img.shape[1], img.shape[2])) # for i in range(self.cfg.NUM_JOINTS - 1): # heatmap[i, :, :] = draw_heatmap(heatmap[i], coor[i], self.cfg.HEATMAP.SIGMA, type = self.cfg.HEATMAP.TYPE) meta = edict({'name': path}) assert coor.min() > 0, path return {'img': img, 'coor': to_torch(coor), 'meta': meta}
def __getitem__(self, idx): name = self.names[idx // 5220] path = self.db[name][idx % 5220] coor = self.anno[name][idx % 5220] img = load_image(path) #calculate ground truth coordination coor = torch.tensor(coor) coor[:, 0] = coor[:, 0] * img.size(1) coor[:, 1] = (1 - coor[:, 1]) * img.size(2) coor = coor[:, :2] #apply transforms into image and calculate cooresponding coor if self.cfg.TRANSFORMS: img, coor = self.transforms(self.cfg.TRANSFORMS, img, coor) #openpose require 22 channel, discard the last one heatmap = np.zeros((self.cfg.NUM_JOINTS, img.shape[1], img.shape[2])) for i in range(self.cfg.NUM_JOINTS - 1): heatmap[i, :, :] = draw_heatmap(heatmap[i], coor[i], self.cfg.HEATMAP.SIGMA, type=self.cfg.HEATMAP.TYPE) # print (name, coor) # fig = plt.figure(1) # ax = fig.add_subplot(111) # plot_hand(im_to_numpy(img), coor, ax) # plt.show() meta = edict({'name': path}) assert coor.min() > 0, path return { 'input': { 'img': img }, 'coor': to_torch(coor), 'heatmap': to_torch(heatmap), 'weight': 1, 'meta': meta }
def reprocess(self, input, reprocess_cfg): img = input['img'] coor2d = input['coor2d'] matrix = input['matrix'] meta = input['meta'] #apply transforms into image and calculate cooresponding coor and camera instrict matrix if reprocess_cfg.TRANSFORMS: img, _, coor2d, matrix = transforms(reprocess_cfg.TRANSFORMS, img, None, coor2d, matrix) matrix = np.linalg.inv(matrix) #take the inversion of matrix coor3d = coor2d.clone() coor3d[:, :2] *= coor3d[:, 2:] coor3d = torch.matmul(coor3d, to_torch(matrix).transpose(0, 1)) root_depth = coor2d[0, 2].clone() index_bone_length = torch.norm(coor3d[9, :] - coor3d[10, :]) relative_depth = (coor2d[:, 2] - root_depth) / index_bone_length heatmap = torch.zeros(reprocess_cfg.NUM_JOINTS, img.size(1), img.size(2)) for i in range(21): heatmap[i] = draw_heatmap(heatmap[i], coor2d[i], reprocess_cfg.HEATMAP.SIGMA) return { 'input': { 'img': img }, 'heatmap': heatmap, 'matrix': to_torch(matrix), 'color_hm': heatmap, 'coor3d': to_torch(coor3d), 'coor2d': to_torch(coor2d), 'root_depth': root_depth, 'index_bone_length': index_bone_length, 'relative_depth': relative_depth, 'weight': 1, 'meta': meta, }
def __getitem__(self, idx): w = self.db[idx] image_path = os.path.join(self.cfg.ROOT, w[1], w[1] + w[2], 'image', w[0] + '.png') label_path = os.path.join(self.cfg.ROOT, w[1], w[1] + w[2], 'label', w[0] + '.json') img = load_image(image_path) label = json.load(open(label_path)) #calculate ground truth coordination coor = torch.tensor(label['camera']).numpy() coor[1:, :] = coor[1:, :].reshape(5, 4, -1)[:, ::-1, :].reshape( 20, -1) #iccv order ! coor = np.array(coor) coor = to_torch(coor) coor = coor - coor[:1, :].repeat(21, 1) index_bone_length = torch.norm(coor[12, :] - coor[11, :]) #apply transforms into image and calculate cooresponding coor if self.cfg.TRANSFORMS: img, label = self.transforms(self.cfg.TRANSFORMS, img, coor) meta = edict({'name': w[1] + ' ' + w[2] + ' ' + w[0]}) return { 'input': { 'img': img, 'hand_side': torch.tensor([0, 1]).float() }, 'coor': to_torch(coor), # 'heat_map': to_torch(heat_map), 'index_bone_length': index_bone_length, 'weight': 1, 'meta': meta }
def __getitem__(self, idx): name = self.names[idx // 5220] path = self.db[name][idx % 5220] coor = self.anno[name][idx % 5220] img = load_image(path) #calculate ground truth coordination coor = torch.tensor(coor).numpy() coor[1:, :] = coor[1:, :].reshape(5, 4, -1)[:, ::-1, :].reshape( 20, -1) #iccv order ! coor = np.array(coor) coor = to_torch(coor) coor = coor - coor[:1, :].repeat(21, 1) index_bone_length = torch.norm(coor[12, :] - coor[11, :]) #apply transforms into image and calculate cooresponding coor if self.cfg.TRANSFORMS: img, label = self.transforms(self.cfg.TRANSFORMS, img, coor) # heat_map = np.zeros((self.cfg.NUM_JOINTS, img.shape[1], img.shape[2])) # for i in range(self.cfg.NUM_JOINTS): # heat_map[i, :, :] = draw_heatmap(heat_map[i], coor[i], self.cfg.HEATMAP.SIGMA, type = self.cfg.HEATMAP.TYPE) meta = edict({'name': path}) return { 'input': { 'img': img, 'hand_side': torch.tensor([0, 1]).float() }, 'coor': to_torch(coor), # 'heat_map': to_torch(heat_map), 'index_bone_length': index_bone_length, 'weight': 1, 'meta': meta }
def __getitem__(self, idx): name = self.name[idx // 1500] coor2d = self.db[name]['sk']['coor2d'][idx % 1500,:,:] matrix = self.db[name]['sk']['matrix'][idx % 1500] name = name.split('_') image_path = os.path.join(self.cfg.ROOT, name[0], 'SK_' + str(idx % 1500) + '.png') img = load_image(image_path, mode = 'RGB') depth_path = os.path.join(self.cfg.ROOT, name[0], 'SK_depth_{}.pickle'.format(idx % 1500)) depthmap = pickle.load(open(depth_path)).unsqueeze(0) # coor2d = label['project'] # assert coor2d[:, :2].min() >= 0 and coor2d[:, :2].max() < 320 # coor2d = torch.matmul(coor3d, to_torch(matrix).transpose(0, 1)) meta = edict({'name': name}) return {'img':img, 'depthmap': depthmap, # 'index': index, 'matrix': to_torch(matrix), 'coor2d': to_torch(coor2d), 'meta': meta, }
def __getitem__(self, idx): name = self.name[idx // 1500] coor = self.db[idx // 1500][:, :, idx % 1500] coor = coor.transpose(1, 0) / 1000. #milionmeter to meter coor[1:, ] = coor[1:, :].reshape(5, 4, -1)[::-1, ::-1, :].reshape(20, -1) coor = np.array(coor) coor = to_torch(coor) index_bone_length = torch.norm(coor[12, :] - coor[11, :]) coor = coor - coor[:1, :].repeat(21, 1) name = name.split("_") if name[1] == 'BB': image_path = os.path.join( self.cfg.ROOT, name[0], "_".join(['left', str(idx % 1500)]) + '.png') elif name[1] == 'SK': image_path = os.path.join( self.cfg.ROOT, name[0], "_".join(['SK', str(idx % 1500)]) + '.png') else: raise Exception("Unrecognized name {}".format(name)) img = load_image(image_path) #apply transforms into image and calculate cooresponding coor if self.cfg.TRANSFORMS: img, coor = self.transforms(self.cfg.TRANSFORMS, img, coor) meta = edict({'name': name}) isleft = 1 return { 'input': { 'img': img, 'hand_side': torch.tensor([isleft, 1 - isleft]).float(), # 'heatmap': heatmap }, "index_bone_length": index_bone_length, 'coor': coor, 'weight': 1, 'meta': meta }
def __getitem__(self, idx): tok = self.db[idx].split('/') image_path = os.path.join(self.cfg.ROOT, tok[0], 'image', tok[1] + '.jpg') img = load_image(image_path, mode='RGB') label_path = os.path.join(self.cfg.ROOT, tok[0], 'annotation', tok[1] + '.torch') label = torch.load(label_path) depth_path = 'data/RHD/cropped/training/depth/' + self.depth_db[ random.randint(0, len(self.depth_db) - 1)] depthmap = torch.load(depth_path).unsqueeze(0) meta = edict({'name': self.db[idx]}) return { 'img': img, 'depthmap': depthmap, 'matrix': to_torch(label['matrix']).float(), 'coor2d': label['coor2d'].float(), 'meta': meta, }
def __getitem__(self, idx): name = self.name[idx // 1500] coor = self.db[idx // 1500][idx % 1500, :, :] coor = to_torch(coor) name = name.split("_") image_path = os.path.join(self.cfg.ROOT, name[0], name[1] + '_' + str(idx % 1500) + '.png') img = load_image(image_path, mode='RGB') #apply transforms into image and calculate cooresponding coor if self.cfg.TRANSFORMS: img, coor = self.transforms(self.cfg.TRANSFORMS, img, coor) # print (name, idx % 1500, coor) # fig = plt.figure(1) # ax = fig.add_subplot(111) # plot_hand(im_to_numpy(img), coor, ax) # plt.show() meta = edict({'name': name}) heatmap = torch.zeros(self.cfg.NUM_JOINTS, img.size(1), img.size(2)) for i in range(self.cfg.NUM_JOINTS - 1): heatmap[i] = draw_heatmap(heatmap[i], coor[i], self.cfg.HEATMAP.SIGMA) return { 'input': { 'img': img }, 'heatmap': heatmap, 'coor': coor, 'weight': 1, 'meta': meta }
def reprocess(input, reprocess_cfg): img = input['img'] depthmap = input['depthmap'] coor2d = input['coor2d'] # assert coor2d[:, :2].min() >= 0 and coor2d[:, :2].max() < 320 matrix = input['matrix'] meta = input['meta'] #apply transforms into image and calculate cooresponding coor and camera instrict matrix if reprocess_cfg.TRANSFORMS: img, depthmap, coor2d, matrix = transforms(reprocess_cfg.TRANSFORMS, img, depthmap, coor2d, matrix) # if depthmap_max - depthmap_min < 1e-6: # print(name, ": ", depthmap_max - depthmap_min) # depthmap = (depthmap.max() - depthmap) / (depthmap_max - depthmap_min) # print(depthmap) matrix = np.linalg.inv(matrix) #take the inversion of matrix coor3d = coor2d.clone() coor3d[:, :2] *= coor3d[:, 2:] coor3d = torch.matmul(coor3d, to_torch(matrix).transpose(0, 1)) root_depth = coor2d[0, 2].clone() index_bone_length = torch.norm(coor3d[9, :] - coor3d[10, :]) relative_depth = (coor2d[:, 2] - root_depth) / index_bone_length depthmap *= float(2**16 - 1) depthmap = (depthmap - root_depth) / index_bone_length depthmap_max = depthmap.max() depthmap_min = depthmap.min() depthmap = (depthmap - depthmap_min) / (depthmap_max - depthmap_min) heatmap = torch.zeros(reprocess_cfg.NUM_JOINTS, img.size(1), img.size(2)) depth = torch.zeros(reprocess_cfg.NUM_JOINTS, img.size(1), img.size(2)) for i in range(21): heatmap[i] = draw_heatmap(heatmap[i], coor2d[i], reprocess_cfg.HEATMAP.SIGMA) depth[i] = heatmap[i] * (coor2d[i, 2] - coor2d[0, 2]) / index_bone_length return { 'input': { 'img': img, 'depthmap': depthmap, }, 'heatmap': heatmap, 'matrix': to_torch(matrix), 'color_hm': heatmap, 'depth_hm': depth, 'depthmap': depthmap, 'depthmap_max': depthmap_max, 'depthmap_range': depthmap_max - depthmap_min, 'coor3d': to_torch(coor3d), 'coor2d': to_torch(coor2d), 'root_depth': root_depth, 'index_bone_length': index_bone_length, 'relative_depth': relative_depth, 'weight': 1, 'meta': meta, }