def load_rpsm_testdata(testdata):
with open(testdata, 'rb') as f:
db = pickle.load(f)
heatmaps = []
cameras = []
boxes = []
poses = []
for i in range(4):
heatmap = db[i]['heatmap'] # [k, h, w]
heatmaps.append(heatmap)
camera = db[i]['cam_params']
cameras.append(camera)
pose_camera = db[i]['joints_3d_cam']
pose_world = camera_to_world_frame(pose_camera, camera['R'],
camera['T'])
poses.append(pose_world)
box = {}
box['scale'] = np.array(db[i]['scale'])
box['center'] = np.array(db[i]['center'])
boxes.append(box)
hms = np.array(heatmaps)
grid_center = poses[0][0]
body = HumanBody()
limb_length = compute_limb_length(body, poses[0])
return cameras, hms, boxes, grid_center, limb_length, poses[0]
def __getitem__(self, idx):
items = self.grouping[idx]
heatmaps = []
boxes = []
poses = []
cameras = []
heatmap_start_idx = self.nviews * idx
for itm_offset, itm in enumerate(items):
datum = self.annot_db[itm]
camera = datum['camera']
cameras.append(camera)
joints_gt = cam_utils.camera_to_world_frame(
datum['joints_3d'], datum['camera']['R'], datum['camera']['T'])
datum['joints_gt'] = joints_gt
# poses.append(datum['joints_gt'])
poses.append(joints_gt)
box = dict()
box['scale'] = np.array(datum['scale'])
box['center'] = np.array(datum['center'])
boxes.append(box)
heatmaps.append(self.heatmaps[heatmap_start_idx + itm_offset])
heatmaps = np.array(heatmaps)
poses = poses[0] # 4 poses are identical
# child - parent, parent is more close to root
bone_vectors = dict()
imu_edges_reverse = self.body.imu_edges_reverse
for boneid in imu_edges_reverse:
parent = imu_edges_reverse[boneid][0]
child = imu_edges_reverse[boneid][1]
bonevec = poses[child] - poses[parent]
bone_vectors[boneid] = bonevec.astype(np.float32)
# # use HumanBody.imubone as bone_vectors dict key
# imubone_mapping = {'Head': 8, 'Pelvis': 2, 'L_UpArm': 11, 'R_UpArm': 13, 'L_LowArm': 12, 'R_LowArm': 14,
# 'L_UpLeg': 5, 'R_UpLeg': 3, 'L_LowLeg': 6, 'R_LowLeg': 4} # todo read from body
# bone_vectors = dict()
# for bone_name in imubone_mapping:
# bone_vectors[imubone_mapping[bone_name]] = bone_vec_tc[bone_name]
limb_length = self.compute_limb_length(self.body, poses)
# todo transform some vars into tensor
# bone_vectors = torch.as_tensor(bone_vectors, dtype=torch.float32)
return {
'heatmaps': heatmaps,
'datum': datum,
'boxes': boxes,
'poses': poses,
'cameras': cameras,
'limb_length': limb_length,
'bone_vectors': bone_vectors
}
def main():
args = parse_args()
logger, final_output_dir, tb_log_dir = create_logger(
config, args.cfg, 'test3d')
prediction_path = os.path.join(final_output_dir,
config.TEST.HEATMAP_LOCATION_FILE)
test_dataset = eval('dataset.' + config.DATASET.TEST_DATASET)(
config, config.DATASET.TEST_SUBSET, False)
all_heatmaps = h5py.File(prediction_path)['heatmaps']
pairwise_file = config.PICT_STRUCT.PAIRWISE_FILE
with open(pairwise_file, 'rb') as f:
pairwise = pickle.load(f)['pairwise_constrain']
cnt = 0
grouping = test_dataset.grouping
mpjpes = []
for items in grouping:
heatmaps = []
boxes = []
poses = []
cameras = []
for idx in items:
datum = test_dataset.db[idx]
camera = datum['camera']
cameras.append(camera)
poses.append(
camera_to_world_frame(datum['joints_3d'], camera['R'],
camera['T']))
box = {}
box['scale'] = np.array(datum['scale'])
box['center'] = np.array(datum['center'])
boxes.append(box)
heatmaps.append(all_heatmaps[cnt])
cnt += 1
heatmaps = np.array(heatmaps)
# This demo uses GT root locations and limb length; but can be replaced by statistics
grid_center = poses[0][0]
body = HumanBody()
limb_length = compute_limb_length(body, poses[0])
prediction = rpsm(cameras, heatmaps, boxes, grid_center, limb_length,
pairwise, config)
mpjpe = np.mean(np.sqrt(np.sum((prediction - poses[0])**2, axis=1)))
mpjpes.append(mpjpe)
print(np.mean(mpjpes))
def load_rpsm_testdata_all(testdata):
with open(testdata, 'rb') as f:
db = pickle.load(f)
assert len(db) % 4 == 0
all_heatmaps = []
all_cameras = []
all_boxes = []
all_grid_centers = []
all_limb_lengths = []
all_gts = []
body = HumanBody()
for idx in range(0,len(db),4):
group_hms = []
group_cameras = []
group_boxes = []
group_gts = []
for inner_idx in range(idx, idx+4):
heatmap = db[inner_idx]['heatmap'] # [16, h, w], body order
group_hms.append(heatmap)
camera = db[inner_idx]['cam_params']
group_cameras.append(camera)
pose_camera = db[inner_idx]['joints_3d_cam'] # [16,3] body order
pose_world = camera_to_world_frame(pose_camera, camera['R'],
camera['T'])
group_gts.append(pose_world)
box = {}
box['scale'] = np.array(db[inner_idx]['scale'])
box['center'] = np.array(db[inner_idx]['center'])
group_boxes.append(box)
group_hms = np.array(group_hms)
all_heatmaps.append(group_hms)
all_cameras.append(group_cameras)
all_boxes.append(group_boxes)
all_grid_centers.append(group_gts[0][body.root_idx])
all_limb_lengths.append(compute_limb_length(body, group_gts[0]))
all_gts.append(group_gts[0])
return all_cameras, all_heatmaps, all_boxes, all_grid_centers, all_limb_lengths, all_gts
def evaluate_3d(self, preds3d, thresholds=None):
if thresholds is None:
thresholds = [
5.,
10.,
15.,
20.,
25.,
50.,
75.,
100.,
125.,
150.,
]
gt3d = []
for idx, items in enumerate(self.grouping):
# note that h36m joints_3d is in camera frame
db_rec = self.db[items[0]]
j3d_global = cam_utils.camera_to_world_frame(
db_rec['joints_3d'], db_rec['camera']['R'],
db_rec['camera']['T'])
gt3d.append(j3d_global)
gt3d = np.array(gt3d)
assert preds3d.shape == gt3d.shape, 'shape mismatch of preds and gt'
distance = np.sum((preds3d - gt3d)**2, axis=2)
num_groupings = len(gt3d)
pcks = []
for thr in thresholds:
detections = distance <= thr**2
detections_perjoint = np.sum(detections, axis=0)
pck_perjoint = detections_perjoint / num_groupings
# pck_avg = np.average(pck_perjoint, axis=0)
pcks.append(pck_perjoint)
return thresholds, pcks
def main():
device = torch.device('cuda:0')
args = parse_args()
logger, final_output_dir, tb_log_dir = create_logger(
config, args.cfg, 'test3d')
prediction_path = os.path.join(final_output_dir,
config.TEST.HEATMAP_LOCATION_FILE)
test_dataset = eval('dataset.' + config.DATASET.TEST_DATASET)(
config, config.DATASET.TEST_SUBSET, False)
all_heatmaps = h5py.File(prediction_path)['heatmaps']
pairwise_file = config.PICT_STRUCT.PAIRWISE_FILE
with open(pairwise_file, 'rb') as f:
pairwise = pickle.load(f)['pairwise_constrain']
for k, v in pairwise.items():
pairwise[k] = torch.as_tensor(v.todense().astype(np.float),
device=device,
dtype=torch.float)
cnt = 0
grouping = test_dataset.grouping
mpjpes = []
for items in grouping:
heatmaps = []
boxes = []
poses = []
cameras = []
for idx in items:
datum = test_dataset.db[idx]
camera = datum['camera']
cameras.append(camera)
poses.append(
camera_to_world_frame(datum['joints_3d'], camera['R'],
camera['T']))
box = {}
box['scale'] = np.array(datum['scale'])
box['center'] = np.array(datum['center'])
boxes.append(box)
heatmaps.append(all_heatmaps[cnt])
cnt += 1
heatmaps = np.array(heatmaps)
grid_center = poses[0][0]
body = HumanBody()
limb_length = compute_limb_length(body, poses[0])
heatmaps = torch.as_tensor(heatmaps, device=device)
kw = {
'body': body,
'boxes': boxes,
'center': grid_center,
'pairwise': pairwise,
'limb_length': limb_length
}
prediction = rpsm(cameras, heatmaps, config, kw)
prediction = prediction.cpu().numpy()
mpjpe = np.mean(np.sqrt(np.sum((prediction - poses[0])**2, axis=1)))
mpjpes.append(mpjpe)
print(mpjpe)
print(np.mean(mpjpe))
def __getitem__(self, idx, source='h36m', **kwargs):
db_rec = copy.deepcopy(self.db[idx])
image_dir = 'images.zip@' if self.data_format == 'zip' else ''
image_file = osp.join(self.root, db_rec['source'], image_dir, 'images',
db_rec['image'])
if self.data_format == 'zip':
from utils import zipreader
data_numpy = zipreader.imread(
image_file, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
else:
data_numpy = cv2.imread(
image_file, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
joints = db_rec['joints_2d'].copy()
joints_vis = db_rec['joints_vis'].copy()
center = np.array(db_rec['center']).copy()
scale = np.array(db_rec['scale']).copy()
rotation = 0
if self.is_train:
sf = self.scale_factor
rf = self.rotation_factor
scale = scale * np.clip(np.random.randn() * sf + 1, 1 - sf, 1 + sf)
rotation = np.clip(np.random.randn() * rf, -rf * 2, rf * 2) \
if random.random() <= 0.6 else 0
trans = get_affine_transform(center, scale, rotation, self.image_size)
input = cv2.warpAffine(
data_numpy,
trans, (int(self.image_size[0]), int(self.image_size[1])),
flags=cv2.INTER_LINEAR)
if self.transform:
input = self.transform(input)
for i in range(self.num_joints):
if joints_vis[i, 0] > 0.0:
joints[i, 0:2] = affine_transform(joints[i, 0:2], trans)
if (np.min(joints[i, :2]) < 0
or joints[i, 0] >= self.image_size[0]
or joints[i, 1] >= self.image_size[1]):
joints_vis[i, :] = 0
target, target_weight = self.generate_target(joints, joints_vis)
target = torch.from_numpy(target)
target_weight = torch.from_numpy(target_weight)
meta = {
'scale': scale,
'center': center,
'rotation': rotation,
'joints_2d': db_rec['joints_2d'],
'joints_2d_transformed': joints,
'joints_vis': joints_vis,
'source': db_rec['source'],
'heatmap_size': self.heatmap_size
}
if source == 'totalcapture':
imubone_mapping = kwargs['tc_imubone_map']
meta['joints_gt'] = db_rec['joints_gt']
meta['bone_vec'] = db_rec['bone_vec']
meta['camera'] = db_rec['camera']
bone_vec_tc = meta['bone_vec']
bone_vectors = dict()
for bone_name in imubone_mapping:
bone_vectors[
imubone_mapping[bone_name]] = bone_vec_tc[bone_name]
meta['bone_vectors'] = bone_vectors
# if self.totalcapture_template_meta is None:
# self.totalcapture_template_meta = meta
elif source == 'h36m':
meta['camera'] = db_rec['camera']
meta['joints_gt'] = cam_utils.camera_to_world_frame(
db_rec['joints_3d'], db_rec['camera']['R'],
db_rec['camera']['T'])
else:
# since tc is mixed with mpii, they should have same keys in meta,
# otherwise will lead to error when collate data in dataloader
meta['joints_gt'] = self.totalcapture_template_meta['joints_gt']
# meta['joints_gt'] = np.zeros((16,3))
meta['bone_vec'] = self.totalcapture_template_meta['bone_vec']
meta['camera'] = self.totalcapture_template_meta['camera']
meta['bone_vectors'] = self.totalcapture_template_meta[
'bone_vectors']
return input, target, target_weight, meta
def main():
args = parse_args()
reset_config(config, args)
no_distortion = True if args.no_distortion else False
test_dataset = eval('dataset.' + config.DATASET.TEST_DATASET)(
config, config.DATASET.TEST_SUBSET, False, pseudo_label_path='', no_distortion=no_distortion)
grouping = test_dataset.grouping
if args.heatmap == '':
flag_test_gt = True
else:
flag_test_gt = False
if not flag_test_gt:
test_data_file = args.heatmap
db = h5py.File(test_data_file, 'r')
pred2d = np.array(db['locations'])[:, :, :2] # [8860, 16, 2]
if flag_test_gt:
pred2d = []
cameras = []
gt3d = []
for items in grouping:
for item in items:
cam = test_dataset.db[item]['camera']
cameras.append(cam)
if flag_test_gt:
pred2d.append(test_dataset.db[item]['joints_2d']) # [N, k_union, 2], already mapped
gt = test_dataset.db[items[-1]]['joints_3d']
gt3d.append(camera_to_world_frame(gt, cameras[-1]['R'], cameras[-1]['T']))
if flag_test_gt:
pred2d = np.array(pred2d)
gt3d = np.array(gt3d)
joints_vis = np.ones(pred2d.shape[:2])
joints_vis = ransac(camera_params=cameras, poses2d=pred2d, joints_vis=joints_vis, config=config) # [N, 16, 3]
pred3d = triangulate_poses(cameras, pred2d, joints_vis=joints_vis, no_distortion=no_distortion) # [N, 16, 3]
assert len(gt3d) == len(pred3d)
u2a = test_dataset.u2a_mapping
u2a = {k:v for k, v in u2a.items() if v != '*'}
sorted_u2a = sorted(u2a.items(), key=lambda x: x[0])
u = np.array([mapping[0] for mapping in sorted_u2a])
a = np.array([mapping[1] for mapping in sorted_u2a])
if flag_test_gt:
compatible_pred = pred3d[:, u, :] # [N, 16, 3]
else:
compatible_pred = pred3d
# compatible_pred = pred3d[:, a, :] # for multivew mixed resutls
compatible_gt = gt3d[:, a, :] # [N, 16, 3]
norm = np.linalg.norm(compatible_pred - compatible_gt, axis=2) # [N, 16]
print('Mean Error:', np.mean(norm))
print('Std Error:', np.std(norm))
print('Max Error:', np.amax(norm))
print('Larger than Mean+Std Error: {:.1%}'.format(np.sum(norm>np.mean(norm)+np.std(norm))/norm.size))
thre_list = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 200, 300, 500, 1000, 2000, 5000, 10000]
print('| ' + ' | '.join(str(thre) for thre in thre_list) + ' |')
print(''.join('| {:.1%} '.format(np.sum(norm<thre)/norm.size) for thre in thre_list) + '|')
def __getitem__(self, idx, source='h36m', **kwargs):
db_rec = copy.deepcopy(self.db[idx])
image_dir = 'images.zip@' if self.data_format == 'zip' else ''
image_file = osp.join(self.root, db_rec['source'], image_dir, 'images',
db_rec['image'])
if self.data_format == 'zip':
from utils import zipreader
data_numpy = zipreader.imread(
image_file, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
else:
data_numpy = cv2.imread(
image_file, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
joints = db_rec['joints_2d'].copy()
joints_vis = db_rec['joints_vis'].copy()
center = np.array(db_rec['center']).copy()
scale = np.array(db_rec['scale']).copy()
rotation = 0
if self.is_train:
sf = self.scale_factor
rf = self.rotation_factor
scale = scale * np.clip(np.random.randn() * sf + 1, 1 - sf, 1 + sf)
rotation = np.clip(np.random.randn() * rf, -rf * 2, rf * 2) \
if random.random() <= 0.6 else 0
trans = get_affine_transform(center, scale, rotation, self.image_size)
# ! Notice: this trans represents full image to cropped image,
# not full image->heatmap
input = cv2.warpAffine(
data_numpy,
trans, (int(self.image_size[0]), int(self.image_size[1])),
flags=cv2.INTER_LINEAR)
if self.transform:
input = self.transform(input)
for i in range(self.num_joints):
if joints_vis[i, 0] > 0.0:
joints[i, 0:2] = affine_transform(joints[i, 0:2], trans)
if (np.min(joints[i, :2]) < 0 or
joints[i, 0] >= self.image_size[0] or
joints[i, 1] >= self.image_size[1]):
joints_vis[i, :] = 0
target, target_weight = self.generate_target(joints, joints_vis)
target = torch.from_numpy(target)
target_weight = torch.from_numpy(target_weight)
# 3x3 data augmentation affine trans (scale rotate)
# !!! Notice: this transformation contains both heatmap->image scale affine
# and data augmentation affine
aug_trans = np.eye(3, 3)
aug_trans[0:2] = trans # full img -> cropped img
hm_scale = self.heatmap_size / self.image_size
scale_trans = np.eye(3,3) # cropped img -> heatmap
scale_trans[0,0] = hm_scale[1]
scale_trans[1, 1] = hm_scale[0]
aug_trans = scale_trans @ aug_trans
meta = {
'scale': scale,
'center': center,
'rotation': rotation,
'joints_2d': db_rec['joints_2d'],
'joints_2d_transformed': joints,
'joints_vis': joints_vis,
'source': db_rec['source'],
'heatmap_size': self.heatmap_size,
'aug_trans': aug_trans,
}
if source == 'totalcapture':
meta['joints_gt'] = db_rec['joints_gt']
meta['camera'] = db_rec['camera']
elif source in ['h36m']:
meta['camera'] = db_rec['camera']
meta['joints_gt'] = cam_utils.camera_to_world_frame(db_rec['joints_3d'], db_rec['camera']['R'], db_rec['camera']['T'])
elif source == 'panoptic':
meta['camera'] = db_rec['camera']
meta['joints_gt'] = db_rec['joints_gt']
elif source in ['unrealcv']:
meta['camera'] = db_rec['camera']
meta['joints_gt'] = db_rec['joints_gt']
else:
assert 0==1, 'No such dataset definition in JointDataset'
return input, target, target_weight, meta