def reset_hessian(self, pose_client, potential_trajectory, use_hessian_mode):
data_list = pose_client.requiredEstimationData
hessian_lift_pose_tensor = pose_client.lift_pose_tensor
if use_hessian_mode == "whole":
self.optimization_mode="estimate_whole"
elif use_hessian_mode == "partial":
self.optimization_mode = "estimate_partial_hessian"
self.projection_client.reset_future(data_list, potential_trajectory, use_hessian_mode)
future_poses = torch.from_numpy(pose_client.future_poses.copy()).float()
if pose_client.USE_LIFT_TERM:
if pose_client.LIFT_METHOD == "complex":
potential_pose3d_lift_directions = calculate_bone_directions(future_poses, np.array(return_lift_bone_connections(self.bone_connections)), batch=True)
if pose_client.LIFT_METHOD == "simple":
potential_pose3d_lift_directions = calculate_bone_directions_simple(future_poses, pose_client.boneLengths, pose_client.BONE_LEN_METHOD, np.array(self.bone_connections), self.hip_index, batch=True)
self.lift_client.reset_future(hessian_lift_pose_tensor, potential_pose3d_lift_directions, use_hessian_mode)
self.pytorch_objective = pytorch_optimizer.pose3d_online_parallel(pose_client, self.projection_client, self.lift_client, optimization_mode=self.optimization_mode)
def determine_relative_3d_pose(pose_client, current_state, my_rng, pose_2d,
cropped_image, heatmap_2d, file_manager):
if not pose_client.USE_LIFT_TERM or pose_client.USE_SINGLE_JOINT or pose_client.is_calibrating_energy:
return None
_, current_pose_3d_GT, _, _, transformation_matrix = current_state.get_frame_parameters(
)
bone_connections, _, _, hip_index = pose_client.model_settings()
if (pose_client.modes["mode_lift"] == 'gt'
or pose_client.modes["mode_lift"] == 'gt_with_noise'):
pose3d_relative = torch.from_numpy(current_pose_3d_GT).clone()
if (pose_client.modes["mode_lift"] == "gt_with_noise"):
pose3d_relative = add_noise_to_pose(pose3d_relative,
my_rng,
noise_type="lift")
elif (pose_client.modes["mode_lift"] == 'lift'):
pose3d_lift = find_lifted_pose(pose_2d, cropped_image,
heatmap_2d.cpu().numpy())
pose3d_relative = pose_client.projection_client.camera_to_world(
pose3d_lift.cpu(), transformation_matrix)
if pose_client.LIFT_METHOD == "complex":
pose3d_lift_directions = calculate_bone_directions(
pose3d_relative,
np.array(return_lift_bone_connections(bone_connections)),
batch=False)
if pose_client.LIFT_METHOD == "simple":
pose3d_lift_directions = calculate_bone_directions_simple(
lift_bones=pose3d_relative,
bone_lengths=pose_client.boneLengths,
bone_length_method=pose_client.BONE_LEN_METHOD,
bone_connections=np.array(bone_connections),
hip_index=hip_index,
batch=False)
# plot_human(pose3d_relative.numpy(), pose3d_relative.numpy(), file_manager.plot_loc, -10, bone_connections, pose_client.USE_SINGLE_JOINT, pose_client.animation, 1000, additional_text = 1000)
# plot_human(pose3d_lift.cpu().numpy(), pose3d_lift.cpu().numpy(), file_manager.plot_loc, -9, bone_connections, pose_client.USE_SINGLE_JOINT, pose_client.animation, 1000, additional_text = 1000)
# plot_human(pose3d_lift_directions.numpy(), pose3d_lift_directions.numpy(), file_manager.plot_loc, -8, bone_connections, pose_client.USE_SINGLE_JOINT, pose_client.animation, 1000, additional_text = 1000)
# plot_human(current_pose_3d_GT-current_pose_3d_GT[:,hip_index][:, np.newaxis], pose3d_lift_directions.numpy(), file_manager.plot_loc, -7, bone_connections, pose_client.USE_SINGLE_JOINT, pose_client.animation, 1000, additional_text = 1000)
#pdb.set_trace()
return pose3d_lift_directions
def determine_relative_3d_pose(pose_client, current_state, my_rng, pose_2d,
cropped_image, heatmap_2d, file_manager):
if not pose_client.USE_LIFT_TERM or pose_client.is_calibrating_energy:
return None
_, current_pose_3d_GT, _, _, transformation_matrix = current_state.get_frame_parameters(
)
bone_connections, _, _, hip_index = pose_client.model_settings()
if (pose_client.modes["mode_lift"] == 'gt'
or pose_client.modes["mode_lift"] == 'gt_with_noise'):
pose3d_relative = torch.from_numpy(current_pose_3d_GT).clone()
if (pose_client.modes["mode_lift"] == "gt_with_noise"):
pose3d_relative = add_noise_to_pose(pose3d_relative,
my_rng,
noise_type="lift",
test_set_name=None)
elif (pose_client.modes["mode_lift"] == 'lift'):
pose3d_lift = find_lifted_pose(pose_2d, cropped_image,
heatmap_2d.cpu().numpy())
pose3d_relative = pose_client.projection_client.camera_to_world(
pose3d_lift.cpu(), transformation_matrix)
if pose_client.LIFT_METHOD == "complex":
pose3d_lift_directions = calculate_bone_directions(
pose3d_relative,
np.array(return_lift_bone_connections(bone_connections)),
batch=False)
if pose_client.LIFT_METHOD == "simple":
pose3d_lift_directions = calculate_bone_directions_simple(
lift_bones=pose3d_relative,
bone_lengths=pose_client.boneLengths,
bone_length_method=pose_client.BONE_LEN_METHOD,
bone_connections=np.array(bone_connections),
hip_index=hip_index,
batch=False)
return pose3d_lift_directions
def forward(self):
output = {}
pose3d_projected = project_trajectory(self.pose3d, self.window_size,
self.num_of_traj_param)
#projection loss
if not self.future_proj:
projected_2d = self.projection_client.take_projection(
pose3d_projected[1:, :, :])
else:
projected_2d = self.projection_client.take_projection(
pose3d_projected)
output["proj"] = mse_loss(projected_2d,
self.projection_client.pose_2d_tensor,
2 * self.NUM_OF_JOINTS)
#smoothness term
if self.smoothness_mode == "velocity":
vel_tensor = pose3d_projected[
1:, :, :] - pose3d_projected[:-1, :, :]
output["smooth"] = mse_loss(vel_tensor[1:, :, :],
vel_tensor[:-1, :, :],
3 * self.NUM_OF_JOINTS)
elif self.smoothness_mode == "position":
output["smooth"] = mse_loss(pose3d_projected[1:, :, :],
pose3d_projected[:-1, :, :],
3 * self.NUM_OF_JOINTS)
elif self.smoothness_mode == "all_connected":
vel_tensor = pose3d_projected[:, :, :] - pose3d_projected[
self.n, :, :]
output["smooth"] = mse_loss(vel_tensor[:, :, :],
vel_tensor[self.n, :, :],
3 * self.NUM_OF_JOINTS)
elif self.smoothness_mode == "only_velo_connected":
vel_tensor = pose3d_projected[
1:, :, :] - pose3d_projected[:-1, :, :]
output["smooth"] = mse_loss(vel_tensor[:, :, :],
vel_tensor[self.m, :, :],
3 * self.NUM_OF_JOINTS)
elif self.smoothness_mode == "none":
output["smooth"] = 0
if self.use_bone_term and not self.use_single_joint:
#bone length consistency
length_of_bone = torch.sum(torch.pow(
pose3d_projected[:, :, self.bone_connections[:, 0]] -
pose3d_projected[:, :, self.bone_connections[:, 1]], 2),
dim=1)
bonelosses = torch.pow((length_of_bone - self.bone_lengths), 2)
output["bone"] = torch.sum(bonelosses) / (self.NUM_OF_JOINTS - 1)
#lift term
if not self.use_single_joint and self.use_lift_term:
if not self.future_proj:
pose_est_directions = calculate_bone_directions(
pose3d_projected[1:, :, :],
self.lift_bone_directions,
batch=True)
else:
pose_est_directions = calculate_bone_directions(
pose3d_projected, self.lift_bone_directions, batch=True)
output["lift"] = mse_loss(self.pose3d_lift_directions,
pose_est_directions,
3 * self.NUM_OF_JOINTS)
overall_output = 0
for loss_key in self.loss_dict:
overall_output += self.energy_weights[loss_key] * output[loss_key]
self.pltpts[loss_key].append(output[loss_key])
return overall_output
def forward(self):
output = {}
#projection loss
if self.optimization_mode == "estimate_future":
output["proj"] = 0
else:
if self.optimization_mode == "estimate_past":
projected_2d = self.projection_client.take_projection(
self.pose3d[self.FUTURE_WINDOW_SIZE:])
proj_N = self.ESTIMATION_WINDOW_SIZE * self.NUM_OF_JOINTS
proj_compare = self.projection_client.pose_2d_tensor
elif self.optimization_mode == "estimate_whole":
projected_2d = self.projection_client.take_projection(
self.pose3d)
proj_N = self.ONLINE_WINDOW_SIZE * self.NUM_OF_JOINTS
proj_compare = self.projection_client.pose_2d_tensor
elif self.optimization_mode == "estimate_partial_hessian":
projected_2d = self.projection_client.take_projection(
self.pose3d[:self.ESTIMATION_WINDOW_SIZE])
proj_N = self.ESTIMATION_WINDOW_SIZE * self.NUM_OF_JOINTS
proj_compare = self.projection_client.pose_2d_tensor
assert proj_compare.shape[0] == self.ESTIMATION_WINDOW_SIZE
output["proj"] = weighted_mse_loss(projected_2d, proj_compare,
self.projection_scales, proj_N)
#bone length consistency
if self.use_bone_term and not self.use_single_joint:
if self.bone_len_method == "no_sqrt":
bone_len_func = calculate_bone_lengths
elif self.bone_len_method == "sqrt":
bone_len_func = calculate_bone_lengths_sqrt
if self.optimization_mode == "estimate_past":
length_of_bone = bone_len_func(
bones=self.pose3d[self.FUTURE_WINDOW_SIZE:],
bone_connections=self.bone_connections,
batch=True)
bone_N = self.ESTIMATION_WINDOW_SIZE * (
length_of_bone.shape[0])
compare_bone_len = self.batch_bone_lengths[self.
FUTURE_WINDOW_SIZE:]
elif self.optimization_mode == "estimate_future":
length_of_bone = bone_len_func(
bones=self.pose3d[:self.FUTURE_WINDOW_SIZE],
bone_connections=self.bone_connections,
batch=True)
bone_N = self.FUTURE_WINDOW_SIZE * (length_of_bone.shape[0])
compare_bone_len = self.batch_bone_lengths[:self.
FUTURE_WINDOW_SIZE, :]
elif self.optimization_mode == "estimate_whole":
length_of_bone = bone_len_func(
bones=self.pose3d,
bone_connections=self.bone_connections,
batch=True)
compare_bone_len = self.batch_bone_lengths
bone_N = self.ONLINE_WINDOW_SIZE * (length_of_bone.shape[0])
elif self.optimization_mode == "estimate_partial_hessian":
length_of_bone = bone_len_func(
bones=self.pose3d[:self.ESTIMATION_WINDOW_SIZE],
bone_connections=self.bone_connections,
batch=True)
compare_bone_len = self.batch_bone_lengths[:self.
ESTIMATION_WINDOW_SIZE]
bone_N = self.ESTIMATION_WINDOW_SIZE * (
length_of_bone.shape[0])
output["bone"] = mse_loss(length_of_bone, compare_bone_len, bone_N)
#smoothness term
if self.smoothness_mode == "velocity":
if self.optimization_mode == "estimate_past":
vel_tensor = self.pose3d[self.FUTURE_WINDOW_SIZE +
1:, :, :] - self.pose3d[
self.FUTURE_WINDOW_SIZE:-1, :, :]
smooth_N = (self.ESTIMATION_WINDOW_SIZE) * self.NUM_OF_JOINTS
elif self.optimization_mode == "estimate_whole" or self.optimization_mode == "estimate_future":
vel_tensor = self.pose3d[1:, :, :] - self.pose3d[:-1, :, :]
smooth_N = (self.ONLINE_WINDOW_SIZE) * self.NUM_OF_JOINTS
elif self.optimization_mode == "estimate_partial_hessian":
vel_tensor = self.pose3d[
1:self.
ESTIMATION_WINDOW_SIZE, :, :] - self.pose3d[:self.
ESTIMATION_WINDOW_SIZE
- 1, :, :]
smooth_N = (self.ESTIMATION_WINDOW_SIZE) * self.NUM_OF_JOINTS
output["smooth"] = mse_loss(vel_tensor[1:, :, :],
vel_tensor[:-1, :, :], smooth_N)
elif self.smoothness_mode == "position":
if self.optimization_mode == "estimate_past":
smooth_N = (self.ESTIMATION_WINDOW_SIZE) * self.NUM_OF_JOINTS
output["smooth"] = mse_loss(
self.pose3d[self.FUTURE_WINDOW_SIZE + 1:, :, :],
self.pose3d[self.FUTURE_WINDOW_SIZE:-1, :, :], smooth_N)
elif self.optimization_mode == "estimate_whole":
smooth_N = (self.ONLINE_WINDOW_SIZE) * self.NUM_OF_JOINTS
output["smooth"] = mse_loss(self.pose3d[1:, :, :],
self.pose3d[:-1, :, :], smooth_N)
elif self.optimization_mode == "estimate_partial_hessian":
smooth_N = (self.ESTIMATION_WINDOW_SIZE) * self.NUM_OF_JOINTS
output["smooth"] = mse_loss(
self.pose3d[1:self.ESTIMATION_WINDOW_SIZE, :, :],
self.pose3d[:self.ESTIMATION_WINDOW_SIZE - 1, :, :],
smooth_N)
elif self.optimization_mode == "estimate_future":
vel_tensor = self.pose3d[1:, :, :] - self.pose3d[:-1, :, :]
smooth_N = (self.ONLINE_WINDOW_SIZE) * self.NUM_OF_JOINTS
output["smooth"] = mse_loss(vel_tensor[1:, :, :],
vel_tensor[:-1, :, :], smooth_N)
elif self.smoothness_mode == "all_connected":
raise NotImplementedError
vel_tensor = self.pose3d[:, :, :] - self.pose3d[self.n, :, :]
output["smooth"] = mse_loss(
vel_tensor[:, :, :], vel_tensor[self.n, :, :],
self.ONLINE_WINDOW_SIZE * self.NUM_OF_JOINTS)
elif self.smoothness_mode == "only_velo_connected":
raise NotImplementedError
vel_tensor = self.pose3d[1:, :, :] - self.pose3d[:-1, :, :]
output["smooth"] = mse_loss(
vel_tensor[:, :, :], vel_tensor[self.m, :, :],
(self.ONLINE_WINDOW_SIZE - 1) * self.NUM_OF_JOINTS)
#lift term
if not self.use_single_joint and self.use_lift_term:
if self.optimization_mode == "estimate_future":
output["lift"] = 0
else:
if self.optimization_mode == "estimate_past":
lift_N = self.ESTIMATION_WINDOW_SIZE * self.NUM_OF_JOINTS
if self.lift_method == "complex":
pose_est_directions = calculate_bone_directions(
self.pose3d[self.FUTURE_WINDOW_SIZE:, :, :],
self.lift_bone_directions,
batch=True)
elif self.lift_method == "simple":
pose_est_direction = self.pose3d[
self.FUTURE_WINDOW_SIZE:] - self.pose3d[
self.FUTURE_WINDOW_SIZE:, :,
self.hip_index].unsqueeze(2)
elif self.optimization_mode == "estimate_whole":
lift_N = self.ONLINE_WINDOW_SIZE * self.NUM_OF_JOINTS
if self.lift_method == "complex":
pose_est_directions = calculate_bone_directions(
self.pose3d, self.lift_bone_directions, batch=True)
elif self.lift_method == "simple":
pose_est_direction = self.pose3d - self.pose3d[:, :,
self.
hip_index].unsqueeze(
2)
elif self.optimization_mode == "estimate_partial_hessian":
lift_N = self.ESTIMATION_WINDOW_SIZE * self.NUM_OF_JOINTS
if self.lift_method == "complex":
pose_est_directions = calculate_bone_directions(
self.pose3d[:self.ESTIMATION_WINDOW_SIZE],
self.lift_bone_directions,
batch=True)
elif self.lift_method == "simple":
pose_est_direction = self.pose3d[:self.
ESTIMATION_WINDOW_SIZE] - self.pose3d[:self
.
ESTIMATION_WINDOW_SIZE, :,
self
.
hip_index].unsqueeze(
2
)
output["lift"] = mse_loss(self.pose3d_lift_directions,
pose_est_direction, lift_N)
overall_output = 0
for loss_key in self.loss_dict:
overall_output += self.energy_weights[loss_key] * output[loss_key]
self.pltpts[loss_key].append(output[loss_key])
self.pltpts_weighted[loss_key].append(
output[loss_key] * self.energy_weights[loss_key])
return overall_output