def test_iteration(self):
self.couple.separate_points_normal_labels()
batchs = self.couple.P1.size(0)
self.couple.add_P2P1(*loss.forward_chamfer(self.network, self.couple.P1, self.couple.P2, local_fix=self.fix,
distChamfer=self.distChamfer))
loss_val_Deformation_ChamferL2 = loss.chamferL2(self.couple.dist1_P2_P1, self.couple.dist2_P2_P1)
loss_val_Reconstruction_L2 = loss.L2(self.couple.P2_P1, self.couple.P2)
self.log.update("loss_val_Deformation_ChamferL2", loss_val_Deformation_ChamferL2)
self.log.update("loss_val_Reconstruction_L2", loss_val_Reconstruction_L2)
print(
'\r' + colored('[%d: %d/%d]' % (self.epoch, self.iteration, self.len_dataset_test / (self.opt.batch_size)),
'red') +
colored('loss_val_Deformation_ChamferL2: %f' % loss_val_Deformation_ChamferL2.item(), 'yellow'),
end='')
if self.iteration % 60 == 1 and self.opt.display:
self.visualizer.show_pointclouds(points=self.couple.P2[0], Y=self.couple.label2[0], title="val_B")
self.visualizer.show_pointclouds(points=self.couple.P1[0], Y=self.couple.label1[0], title="val_A")
self.visualizer.show_pointclouds(points=self.couple.P2_P1[0], Y=self.couple.label1[0],
title="val_B_reconstructed")
# Compute Miou when labels are tranfered from P1 to P2.
predicted_target = self.couple.label1.view(-1)[self.couple.idx2_P2_P1].view(batchs, -1)
for shape in range(batchs):
if self.couple.cat_1 == self.couple.cat_2:
target = self.couple.label2[shape].squeeze().data.cpu().numpy()
iou_val = miou_shape.miou_shape(predicted_target[shape].squeeze().cpu().numpy(), target,
self.dataset_train.part_category[self.couple.cat_1[shape]])
self.log.update("iou_val", iou_val)
try:
P2, _, _, P2_path = trainer.dataset_test[i]
P2_label = P2[:, 6].data.cpu().numpy()
P2 = P2[:, :3].unsqueeze(0).contiguous().cuda().float()
P2_latent = trainer.network.encode(
P2.transpose(1, 2).contiguous(),
P2.transpose(1, 2).contiguous())
except:
break
# Chamfer (P0_P2)
P0_P2_list = list(
map(
lambda x: loss.forward_chamfer(trainer.network,
P2,
x,
local_fix=None,
distChamfer=distChamfer),
points_train_list))
# Ensemble method
def emsemble(top_k_idx, predicted_list):
predicted_emsemble = predicted_list[top_k_idx]
Ensemble = my_utils.get_emsemble(predicted_emsemble)
iou = miou_shape.miou_shape(Ensemble, P2_label, trainer.parts)
return iou
iou_dict = {}
def add(name, top, ensemble=True, ensemble_list=None):
iou_dict[name + "_NN"] = iou_NN_list[top[0]]
def deform_source_in_target(self, i, top, name):
"""
:param i: ranking of source,target pair in top_cycle
:param top: index of source,target pair in top_cycle
:return: save meshes of deformation and label transfer
top_cycles: {
idx : idx of best train sammple for target
values : value of cycle criterion for source/target pair
iou_ours : iou obtained with our method for source/target pairs
iou_NN : iou of the nearest neighbors for source/target pairs
predicted_NN_P2_P0 : label indexes of target with the nearest neighbors for source/target pairs
predicted_ours_P2_P0 : label indexes of target with our methodfor source/target pairs
P2_P0 : deformed source in target
P2 : target
P2_label : target original labels
P2_path : Path to target
"""
idx, values, iou_ours, iou_NN, predicted_NN_P2_P0, predicted_ours_P2_P0, P2_P0, P2, P2_label, P2_path = \
self.top_cycles[top]
print("computing results for ...", P2_path)
train_path = self.path_train_list[idx]
mesh_path = convert_path(self.opt.shapenetv1_path, train_path)
source_mesh_edge = get_shapenet_model.link(mesh_path)
pymesh.save_mesh(
os.path.join(self.figure_folder, name + "_" + str(i) + "_SourceMesh_" + train_path[33:-4] + ".ply"),
source_mesh_edge, ascii=True)
mesh_path = convert_path(self.opt.shapenetv1_path, P2_path)
target_mesh_edge = get_shapenet_model.link(mesh_path)
pymesh.save_mesh(
os.path.join(self.figure_folder, name + "_" + str(i) + "_TargetMesh_" + P2_path[33:-4] + ".ply"),
target_mesh_edge, ascii=True)
with torch.no_grad():
P2_P1_mesh, _, _, _, _ = loss.forward_chamfer(self.trainer.network, torch.from_numpy(
source_mesh_edge.vertices).cuda().float().unsqueeze(0), torch.from_numpy(
target_mesh_edge.vertices).cuda().float().unsqueeze(0), local_fix=None, distChamfer=self.distChamfer)
P2_P1_mesh = pymesh.form_mesh(vertices=P2_P1_mesh.squeeze().cpu().numpy(), faces=source_mesh_edge.faces)
pymesh.save_mesh(
os.path.join(self.figure_folder, name + "_" + str(i) + "_SourceMeshDeformed_" + train_path[33:-4] + ".ply"),
P2_P1_mesh, ascii=True)
high_frequencies.high_frequency_propagation(
os.path.join(self.figure_folder, name + "_" + str(i) + "_SourceMesh_" + train_path[33:-4] + ".ply"),
os.path.join(self.figure_folder, name + "_" + str(i) + "_SourceMeshDeformed_" + train_path[33:-4] + ".ply"),
os.path.join(self.figure_folder, name + "_" + str(i) + "_TargetMesh_" + P2_path[33:-4] + ".ply"))
smfpl.save_mesh_from_pointsandlabels(self.points_train_list[idx], self.labels_train_list[idx],
os.path.join(self.figure_folder,
name + "_" + str(i) + "_SourcePoints_" + train_path[
33:-4] + ".ply"),
parts=self.parts)
smfpl.save_mesh_from_pointsandlabels(P2, P2_label,
os.path.join(self.figure_folder,
name + "_" + str(i) + "_TargetPoints_" + P2_path[
33:-4] + ".ply"),
parts=self.parts)
smfpl.save_mesh_from_pointsandlabels(P2_P0[0], self.labels_train_list[idx],
os.path.join(self.figure_folder,
name + "_" + str(i) + "_SourcePointsDeformed_" + P2_path[
33:-4] + ".ply"),
parts=self.parts)
smfpl.save_mesh_from_pointsandlabels(P2, predicted_ours_P2_P0,
os.path.join(self.figure_folder, name + "_" + str(
i) + "_TargetPointsPredictedLabelsOURS_" + P2_path[
33:-4] + ".ply"),
parts=self.parts)
smfpl.save_mesh_from_pointsandlabels(P2, predicted_NN_P2_P0,
os.path.join(self.figure_folder, name + "_" + str(
i) + "_TargetPointsPredictedLabelsNN_" + P2_path[
33:-4] + ".ply"),
parts=self.parts)
with open(os.path.join(self.figure_folder, name + "_" + str(i) + "_stats.txt"), 'w') as fp:
fp.write(f"index {name} train sample : " + str(idx) + '\n')
fp.write(f"value {name} train sample : " + '%.2f' % (values * 100) + '\n')
fp.write(f"iou_ours {name} train sample : " + '%.2f' % (iou_ours * 100) + '\n')
fp.write(f"iou_NN {name} train sample : " + '%.2f' % (iou_NN * 100) + '\n')
def get_criterion_shape(opt):
return_dict = {}
my_utils.plant_seeds(randomized_seed=opt.randomize)
trainer = t.Trainer(opt)
trainer.build_dataset_train_for_matching()
trainer.build_dataset_test_for_matching()
trainer.build_network()
trainer.build_losses()
trainer.network.eval()
# Load input mesh
exist_P2_label = True
try:
mesh_path = opt.eval_get_criterions_for_shape # Ends in .txt
points = np.loadtxt(mesh_path)
points = torch.from_numpy(points).float()
# Normalization is done before resampling !
P2 = normalize_points.BoundingBox(points[:, :3])
P2_label = points[:, 6].data.cpu().numpy()
except:
mesh_path = opt.eval_get_criterions_for_shape # Ends in .obj
source_mesh_edge = get_shapenet_model.link(mesh_path)
P2 = torch.from_numpy(source_mesh_edge.vertices)
exist_P2_label = False
min_k = Min_k(opt.k_max_eval)
max_k = Max_k(opt.k_max_eval)
points_train_list = []
point_train_paths = []
labels_train_list = []
iterator_train = trainer.dataloader_train.__iter__()
for find_best in range(opt.num_shots_eval):
try:
points_train, _, _, file_path = iterator_train.next()
points_train_list.append(
points_train[:, :, :3].contiguous().cuda().float())
point_train_paths.append(file_path)
labels_train_list.append(
points_train[:, :, 6].contiguous().cuda().float())
except:
break
# ========Loop on test examples======================== #
with torch.no_grad():
P2 = P2[:, :3].unsqueeze(0).contiguous().cuda().float()
P2_latent = trainer.network.encode(
P2.transpose(1, 2).contiguous(),
P2.transpose(1, 2).contiguous())
# Chamfer (P0_P2)
P0_P2_list = list(
map(
lambda x: loss.forward_chamfer(trainer.network,
P2,
x,
local_fix=None,
distChamfer=trainer.distChamfer
), points_train_list))
# Compute Chamfer (P2_P0)
P2_P0_list = list(
map(
lambda x: loss.forward_chamfer(trainer.network,
x,
P2,
local_fix=None,
distChamfer=trainer.distChamfer
), points_train_list))
predicted_ours_P2_P0_list = list(
map(lambda x, y: x.view(-1)[y[4].view(-1).data.long()].view(1, -1),
labels_train_list, P2_P0_list))
if exist_P2_label:
iou_ours_list = list(
map(
lambda x: miou_shape.miou_shape(x.squeeze().cpu().numpy(
), P2_label, trainer.parts), predicted_ours_P2_P0_list))
top_k_idx, top_k_values = max_k(iou_ours_list)
return_dict["oracle"] = point_train_paths[top_k_idx[0]][0]
predicted_ours_P2_P0_list = list(
map(lambda x, y: x.view(-1)[y[4].view(-1).data.long()].view(1, -1),
labels_train_list, P2_P0_list))
predicted_ours_P2_P0_list = torch.cat(predicted_ours_P2_P0_list)
# Compute NN
P2_P0_NN_list = list(
map(lambda x: loss.distChamfer(x, P2), points_train_list))
predicted_NN_P2_P0_list = list(
map(lambda x, y: x.view(-1)[y[3].view(-1).data.long()].view(1, -1),
labels_train_list, P2_P0_NN_list))
predicted_NN_P2_P0_list = torch.cat(predicted_NN_P2_P0_list)
# NN
NN_chamferL2_list = list(
map(lambda x: loss.chamferL2(x[0], x[1]), P2_P0_NN_list))
top_k_idx, top_k_values = min_k(NN_chamferL2_list)
return_dict["NN_criterion"] = point_train_paths[top_k_idx[0]][0]
# Chamfer ours
chamfer_list = list(
map(lambda x: loss.chamferL2(x[1], x[2]), P2_P0_list))
top_k_idx, top_k_values = min_k(chamfer_list)
return_dict["chamfer_criterion"] = point_train_paths[top_k_idx[0]][0]
# NN in latent space
P0_latent_list = list(
map(
lambda x: trainer.network.encode(
x.transpose(1, 2).contiguous(),
x.transpose(1, 2).contiguous()), points_train_list))
cosine_list = list(
map(lambda x: loss.cosine(x, P2_latent), P0_latent_list))
top_k_idx, top_k_values = min_k(cosine_list)
return_dict["cosine_criterion"] = point_train_paths[top_k_idx[0]][0]
# Cycle 2
P0_P2_cycle_list = list(
map(lambda x, y: loss.batch_cycle_2(x[0], y[3], 1), P0_P2_list,
P2_P0_list))
P0_P2_cycle_list = list(
map(lambda x, y: loss.L2(x, y), P0_P2_cycle_list,
points_train_list))
P2_P0_cycle_list = list(
map(lambda x, y: loss.batch_cycle_2(x[0], y[3], 1), P2_P0_list,
P0_P2_list))
P2_P0_cycle_list = list(map(lambda x: loss.L2(x, P2),
P2_P0_cycle_list))
# Cycle 2 both sides
both_cycle_list = list(
map(lambda x, y: x * y, P0_P2_cycle_list, P2_P0_cycle_list))
both_cycle_list = np.power(both_cycle_list, 1.0 / 2.0).tolist()
top_k_cycle2_idx, top_k_values = min_k(both_cycle_list)
return_dict["cycle_criterion"] = point_train_paths[
top_k_cycle2_idx[0]][0]
pprint.pprint(return_dict)
return return_dict
def forward(opt):
"""
Takes an input and a target mesh. Deform input in output and propagate a
manually defined high frequency from the oinput to the output
:return:
"""
my_utils.plant_seeds(randomized_seed=opt.randomize)
os.makedirs(opt.output_dir, exist_ok=True)
trainer = t.Trainer(opt)
trainer.build_dataset_train_for_matching()
trainer.build_dataset_test_for_matching()
trainer.build_network()
trainer.build_losses()
trainer.network.eval()
if opt.eval_list and os.path.isfile(opt.eval_list):
source_target_files = np.loadtxt(opt.eval_list, dtype=str)
source_target_files = source_target_files.tolist()
for i, st in enumerate(source_target_files):
source, target = st
cat1, fname1 = source.split('/')
fname1 = os.path.splitext(fname1)[0]
cat2, fname2 = target.split('/')
fname2 = os.path.splitext(fname2)[0]
if len(opt.shapenetv1_path) > 0:
source_target_files[i] = (os.path.join(opt.shapenetv1_path,
cat1, fname1,
"model.obj"),
os.path.join(opt.shapenetv1_path,
cat2, fname2,
"model.obj"))
elif len(opt.shapenetv2_path) > 0:
source_target_files[i] = (os.path.join(opt.shapenetv2_path,
cat1, fname1, "models",
"model_normalized.obj"),
os.path.join(opt.shapenetv2_path,
cat2, fname2, "models",
"model_normalized.obj"))
elif (opt.eval_source != "" and opt.eval_source[-4:] == ".txt") and (
opt.eval_target != "" and opt.eval_target[-4:] == ".txt"):
source_target_files = [
(figure_2_3.convert_path(opt.shapenetv1_path, opt.eval_source),
figure_2_3.convert_path(opt.shapenetv1_path, opt.eval_target))
]
rot_mat = get_3D_rot_matrix(1, np.pi / 2)
rot_mat_rev = get_3D_rot_matrix(1, -np.pi / 2)
isV2 = len(opt.shapenetv2_path) > 0
for i, source_target in enumerate(source_target_files):
basename = get_model_id(source_target[0], isV2) + "-" + get_model_id(
source_target[1], isV2)
path_deformed = os.path.join(opt.output_dir, basename + "-Sab.ply")
path_source = os.path.join(opt.output_dir, basename + "-Sa.ply")
path_target = os.path.join(opt.output_dir, basename + "-Sb.ply")
mesh_path = source_target[0]
print(mesh_path)
source_mesh_edge = get_shapenet_model.link(mesh_path)
mesh_path = source_target[1]
target_mesh_edge = get_shapenet_model.link(mesh_path)
print("Deforming source in target")
source = source_mesh_edge.vertices
target = target_mesh_edge.vertices
pymesh.save_mesh_raw(path_source,
source,
source_mesh_edge.faces,
ascii=True)
pymesh.save_mesh_raw(path_target,
target,
target_mesh_edge.faces,
ascii=True)
if len(opt.shapenetv2_path) > 0:
source = source.dot(rot_mat)
target = target.dot(rot_mat)
source = torch.from_numpy(source).cuda().float().unsqueeze(0)
target = torch.from_numpy(target).cuda().float().unsqueeze(0)
with torch.no_grad():
source, _, _, _, _ = loss.forward_chamfer(
trainer.network,
source,
target,
local_fix=None,
distChamfer=trainer.distChamfer)
try:
source = source.squeeze().cpu().detach().numpy()
if len(opt.shapenetv2_path) > 0:
source = source.dot(rot_mat_rev)
P2_P1_mesh = pymesh.form_mesh(vertices=source,
faces=source_mesh_edge.faces)
pymesh.save_mesh(path_deformed, P2_P1_mesh, ascii=True)
# print("computing signal tranfer form source to target")
# high_frequencies.high_frequency_propagation(path_source, path_deformed, path_target)
except Exception as e:
print(e)
import pdb
pdb.set_trace()
path_deformed = path_deformed[:-4] + ".pts"
save_pts(path_deformed, source.squeeze().cpu().detach().numpy())
# Nearest Neighbor (NN):
# identity_nn = distChamfer(src_points.unsqueeze(0).cuda(), tgt_points.unsqueeze(0).cuda())
# identity_pred_labels = src_labels[identity_nn[3].view(-1).cpu().data.long()]
# visualize_points(tgt_points.cpu().numpy(), bound=1.0, c=identity_pred_labels.cpu().numpy(), out_file=os.path.join(figures_folder, 'tgt_' + str(j) + '_nn.png'))
# ICP + NN:
# icp_src_points = ICP.ICP(src_points.unsqueeze(0), tgt_points.unsqueeze(0))
# visualize_points(icp_src_points.cpu().numpy(), bound=1.0, c=src_labels.cpu().numpy(), out_file=os.path.join(figures_folder, 'tgt_' + str(j) + '_icp_inter.png'))
# icp_nn = distChamfer(icp_src_points.unsqueeze(0).cuda(), tgt_points.unsqueeze(0).cuda())
# icp_pred_labels = src_labels[icp_nn[3].view(-1).cpu().data.long()]
# visualize_points(tgt_points.cpu().numpy(), bound=1.0, c=icp_pred_labels.cpu().numpy(), out_file=os.path.join(figures_folder, 'tgt_' + str(j) + '_icp.png'))
# Cycle Consistency model:
cc_forward = loss.forward_chamfer(trainer.network,
src_points.unsqueeze(0).cuda(),
tgt_points.unsqueeze(0).cuda(),
local_fix=None,
distChamfer=distChamfer)
cc_src_points = cc_forward[0].squeeze(0)
visualize_points(cc_src_points.cpu().numpy(),
bound=1.0,
c=src_labels.cpu().numpy(),
out_file=os.path.join(
figures_folder,
'tgt_' + str(j) + '_cc_inter.png'),
show=True)
cc_pred_labels = src_labels[cc_forward[4].view(
-1).cpu().data.long()]
visualize_points(tgt_points.cpu().numpy(),
bound=1.0,
c=cc_pred_labels,
def forward(opt):
"""
Takes an input and a target mesh. Deform input in output and propagate a
manually defined high frequency from the oinput to the output
:return:
"""
my_utils.plant_seeds(randomized_seed=opt.randomize)
pdb.set_trace()
trainer = t.Trainer(opt)
trainer.build_dataset_train_for_matching()
trainer.build_dataset_test_for_matching()
trainer.build_network()
trainer.build_losses()
trainer.network.eval()
if opt.eval_source[-4:] == ".txt":
opt.eval_source = figure_2_3.convert_path(opt.shapenetv1_path,
opt.eval_source)
if opt.eval_target[-4:] == ".txt":
opt.eval_target = figure_2_3.convert_path(opt.shapenetv1_path,
opt.eval_target)
path_deformed = os.path.join("./figures/forward_input_target/",
opt.eval_source[-42:-10] + "deformed.ply")
path_source = os.path.join("./figures/forward_input_target/",
opt.eval_source[-42:-10] + ".ply")
path_target = os.path.join(
"./figures/forward_input_target/",
opt.eval_source[-42:-10] + "_" + opt.eval_target[-42:-10] + ".ply")
mesh_path = opt.eval_source
print(mesh_path)
source_mesh_edge = get_shapenet_model.link(mesh_path)
mesh_path = opt.eval_target
target_mesh_edge = get_shapenet_model.link(mesh_path)
pymesh.save_mesh(path_source, source_mesh_edge, ascii=True)
pymesh.save_mesh(path_target, target_mesh_edge, ascii=True)
print("Deforming source in target")
source = torch.from_numpy(
source_mesh_edge.vertices).cuda().float().unsqueeze(0)
target = torch.from_numpy(
target_mesh_edge.vertices).cuda().float().unsqueeze(0)
with torch.no_grad():
source, _, _, _, _ = loss.forward_chamfer(
trainer.network,
source,
target,
local_fix=None,
distChamfer=trainer.distChamfer)
P2_P1_mesh = pymesh.form_mesh(
vertices=source.squeeze().cpu().detach().numpy(),
faces=source_mesh_edge.faces)
pymesh.save_mesh(path_deformed, P2_P1_mesh, ascii=True)
print("computing signal tranfer form source to target")
high_frequencies.high_frequency_propagation(path_source, path_deformed,
path_target)
with torch.no_grad():
for i in tqdm.tqdm(range(trainer.len_dataset_test)):
try:
P1, _, _, _, P2, _, _, _ = trainer.dataset_test[i]
# P2, _, _, _ = trainer.dataset_test[i]
P1 = torch.tensor(
P1[:, :3]).unsqueeze(0).contiguous().cuda().float()
P2 = torch.tensor(
P2[:, :3]).unsqueeze(0).contiguous().cuda().float()
except:
break
visualize_points(P1[0].cpu().numpy(), bound=0.5, show=True)
visualize_points(P2[0].cpu().numpy(), bound=0.5, show=True)
# pdb.set_trace()
P1_2 = loss.forward_chamfer(trainer.network,
P1,
P2,
local_fix=None,
distChamfer=distChamfer)
visualize_points(P1_2[0][0].cpu().numpy(), bound=0.5, show=True)
visualize_points_overlay(
[P1_2[0][0].cpu().numpy(), P2[0].cpu().numpy()],
bound=0.5,
show=True)
