def test_iteration(self): self.couple.separate_points_normal_labels() batchs = self.couple.P1.size(0) self.couple.add_P2P1(*loss.forward_chamfer(self.network, self.couple.P1, self.couple.P2, local_fix=self.fix, distChamfer=self.distChamfer)) loss_val_Deformation_ChamferL2 = loss.chamferL2(self.couple.dist1_P2_P1, self.couple.dist2_P2_P1) loss_val_Reconstruction_L2 = loss.L2(self.couple.P2_P1, self.couple.P2) self.log.update("loss_val_Deformation_ChamferL2", loss_val_Deformation_ChamferL2) self.log.update("loss_val_Reconstruction_L2", loss_val_Reconstruction_L2) print( '\r' + colored('[%d: %d/%d]' % (self.epoch, self.iteration, self.len_dataset_test / (self.opt.batch_size)), 'red') + colored('loss_val_Deformation_ChamferL2: %f' % loss_val_Deformation_ChamferL2.item(), 'yellow'), end='') if self.iteration % 60 == 1 and self.opt.display: self.visualizer.show_pointclouds(points=self.couple.P2[0], Y=self.couple.label2[0], title="val_B") self.visualizer.show_pointclouds(points=self.couple.P1[0], Y=self.couple.label1[0], title="val_A") self.visualizer.show_pointclouds(points=self.couple.P2_P1[0], Y=self.couple.label1[0], title="val_B_reconstructed") # Compute Miou when labels are tranfered from P1 to P2. predicted_target = self.couple.label1.view(-1)[self.couple.idx2_P2_P1].view(batchs, -1) for shape in range(batchs): if self.couple.cat_1 == self.couple.cat_2: target = self.couple.label2[shape].squeeze().data.cpu().numpy() iou_val = miou_shape.miou_shape(predicted_target[shape].squeeze().cpu().numpy(), target, self.dataset_train.part_category[self.couple.cat_1[shape]]) self.log.update("iou_val", iou_val)
try: P2, _, _, P2_path = trainer.dataset_test[i] P2_label = P2[:, 6].data.cpu().numpy() P2 = P2[:, :3].unsqueeze(0).contiguous().cuda().float() P2_latent = trainer.network.encode( P2.transpose(1, 2).contiguous(), P2.transpose(1, 2).contiguous()) except: break # Chamfer (P0_P2) P0_P2_list = list( map( lambda x: loss.forward_chamfer(trainer.network, P2, x, local_fix=None, distChamfer=distChamfer), points_train_list)) # Ensemble method def emsemble(top_k_idx, predicted_list): predicted_emsemble = predicted_list[top_k_idx] Ensemble = my_utils.get_emsemble(predicted_emsemble) iou = miou_shape.miou_shape(Ensemble, P2_label, trainer.parts) return iou iou_dict = {} def add(name, top, ensemble=True, ensemble_list=None): iou_dict[name + "_NN"] = iou_NN_list[top[0]]
def deform_source_in_target(self, i, top, name): """ :param i: ranking of source,target pair in top_cycle :param top: index of source,target pair in top_cycle :return: save meshes of deformation and label transfer top_cycles: { idx : idx of best train sammple for target values : value of cycle criterion for source/target pair iou_ours : iou obtained with our method for source/target pairs iou_NN : iou of the nearest neighbors for source/target pairs predicted_NN_P2_P0 : label indexes of target with the nearest neighbors for source/target pairs predicted_ours_P2_P0 : label indexes of target with our methodfor source/target pairs P2_P0 : deformed source in target P2 : target P2_label : target original labels P2_path : Path to target """ idx, values, iou_ours, iou_NN, predicted_NN_P2_P0, predicted_ours_P2_P0, P2_P0, P2, P2_label, P2_path = \ self.top_cycles[top] print("computing results for ...", P2_path) train_path = self.path_train_list[idx] mesh_path = convert_path(self.opt.shapenetv1_path, train_path) source_mesh_edge = get_shapenet_model.link(mesh_path) pymesh.save_mesh( os.path.join(self.figure_folder, name + "_" + str(i) + "_SourceMesh_" + train_path[33:-4] + ".ply"), source_mesh_edge, ascii=True) mesh_path = convert_path(self.opt.shapenetv1_path, P2_path) target_mesh_edge = get_shapenet_model.link(mesh_path) pymesh.save_mesh( os.path.join(self.figure_folder, name + "_" + str(i) + "_TargetMesh_" + P2_path[33:-4] + ".ply"), target_mesh_edge, ascii=True) with torch.no_grad(): P2_P1_mesh, _, _, _, _ = loss.forward_chamfer(self.trainer.network, torch.from_numpy( source_mesh_edge.vertices).cuda().float().unsqueeze(0), torch.from_numpy( target_mesh_edge.vertices).cuda().float().unsqueeze(0), local_fix=None, distChamfer=self.distChamfer) P2_P1_mesh = pymesh.form_mesh(vertices=P2_P1_mesh.squeeze().cpu().numpy(), faces=source_mesh_edge.faces) pymesh.save_mesh( os.path.join(self.figure_folder, name + "_" + str(i) + "_SourceMeshDeformed_" + train_path[33:-4] + ".ply"), P2_P1_mesh, ascii=True) high_frequencies.high_frequency_propagation( os.path.join(self.figure_folder, name + "_" + str(i) + "_SourceMesh_" + train_path[33:-4] + ".ply"), os.path.join(self.figure_folder, name + "_" + str(i) + "_SourceMeshDeformed_" + train_path[33:-4] + ".ply"), os.path.join(self.figure_folder, name + "_" + str(i) + "_TargetMesh_" + P2_path[33:-4] + ".ply")) smfpl.save_mesh_from_pointsandlabels(self.points_train_list[idx], self.labels_train_list[idx], os.path.join(self.figure_folder, name + "_" + str(i) + "_SourcePoints_" + train_path[ 33:-4] + ".ply"), parts=self.parts) smfpl.save_mesh_from_pointsandlabels(P2, P2_label, os.path.join(self.figure_folder, name + "_" + str(i) + "_TargetPoints_" + P2_path[ 33:-4] + ".ply"), parts=self.parts) smfpl.save_mesh_from_pointsandlabels(P2_P0[0], self.labels_train_list[idx], os.path.join(self.figure_folder, name + "_" + str(i) + "_SourcePointsDeformed_" + P2_path[ 33:-4] + ".ply"), parts=self.parts) smfpl.save_mesh_from_pointsandlabels(P2, predicted_ours_P2_P0, os.path.join(self.figure_folder, name + "_" + str( i) + "_TargetPointsPredictedLabelsOURS_" + P2_path[ 33:-4] + ".ply"), parts=self.parts) smfpl.save_mesh_from_pointsandlabels(P2, predicted_NN_P2_P0, os.path.join(self.figure_folder, name + "_" + str( i) + "_TargetPointsPredictedLabelsNN_" + P2_path[ 33:-4] + ".ply"), parts=self.parts) with open(os.path.join(self.figure_folder, name + "_" + str(i) + "_stats.txt"), 'w') as fp: fp.write(f"index {name} train sample : " + str(idx) + '\n') fp.write(f"value {name} train sample : " + '%.2f' % (values * 100) + '\n') fp.write(f"iou_ours {name} train sample : " + '%.2f' % (iou_ours * 100) + '\n') fp.write(f"iou_NN {name} train sample : " + '%.2f' % (iou_NN * 100) + '\n')
def get_criterion_shape(opt): return_dict = {} my_utils.plant_seeds(randomized_seed=opt.randomize) trainer = t.Trainer(opt) trainer.build_dataset_train_for_matching() trainer.build_dataset_test_for_matching() trainer.build_network() trainer.build_losses() trainer.network.eval() # Load input mesh exist_P2_label = True try: mesh_path = opt.eval_get_criterions_for_shape # Ends in .txt points = np.loadtxt(mesh_path) points = torch.from_numpy(points).float() # Normalization is done before resampling ! P2 = normalize_points.BoundingBox(points[:, :3]) P2_label = points[:, 6].data.cpu().numpy() except: mesh_path = opt.eval_get_criterions_for_shape # Ends in .obj source_mesh_edge = get_shapenet_model.link(mesh_path) P2 = torch.from_numpy(source_mesh_edge.vertices) exist_P2_label = False min_k = Min_k(opt.k_max_eval) max_k = Max_k(opt.k_max_eval) points_train_list = [] point_train_paths = [] labels_train_list = [] iterator_train = trainer.dataloader_train.__iter__() for find_best in range(opt.num_shots_eval): try: points_train, _, _, file_path = iterator_train.next() points_train_list.append( points_train[:, :, :3].contiguous().cuda().float()) point_train_paths.append(file_path) labels_train_list.append( points_train[:, :, 6].contiguous().cuda().float()) except: break # ========Loop on test examples======================== # with torch.no_grad(): P2 = P2[:, :3].unsqueeze(0).contiguous().cuda().float() P2_latent = trainer.network.encode( P2.transpose(1, 2).contiguous(), P2.transpose(1, 2).contiguous()) # Chamfer (P0_P2) P0_P2_list = list( map( lambda x: loss.forward_chamfer(trainer.network, P2, x, local_fix=None, distChamfer=trainer.distChamfer ), points_train_list)) # Compute Chamfer (P2_P0) P2_P0_list = list( map( lambda x: loss.forward_chamfer(trainer.network, x, P2, local_fix=None, distChamfer=trainer.distChamfer ), points_train_list)) predicted_ours_P2_P0_list = list( map(lambda x, y: x.view(-1)[y[4].view(-1).data.long()].view(1, -1), labels_train_list, P2_P0_list)) if exist_P2_label: iou_ours_list = list( map( lambda x: miou_shape.miou_shape(x.squeeze().cpu().numpy( ), P2_label, trainer.parts), predicted_ours_P2_P0_list)) top_k_idx, top_k_values = max_k(iou_ours_list) return_dict["oracle"] = point_train_paths[top_k_idx[0]][0] predicted_ours_P2_P0_list = list( map(lambda x, y: x.view(-1)[y[4].view(-1).data.long()].view(1, -1), labels_train_list, P2_P0_list)) predicted_ours_P2_P0_list = torch.cat(predicted_ours_P2_P0_list) # Compute NN P2_P0_NN_list = list( map(lambda x: loss.distChamfer(x, P2), points_train_list)) predicted_NN_P2_P0_list = list( map(lambda x, y: x.view(-1)[y[3].view(-1).data.long()].view(1, -1), labels_train_list, P2_P0_NN_list)) predicted_NN_P2_P0_list = torch.cat(predicted_NN_P2_P0_list) # NN NN_chamferL2_list = list( map(lambda x: loss.chamferL2(x[0], x[1]), P2_P0_NN_list)) top_k_idx, top_k_values = min_k(NN_chamferL2_list) return_dict["NN_criterion"] = point_train_paths[top_k_idx[0]][0] # Chamfer ours chamfer_list = list( map(lambda x: loss.chamferL2(x[1], x[2]), P2_P0_list)) top_k_idx, top_k_values = min_k(chamfer_list) return_dict["chamfer_criterion"] = point_train_paths[top_k_idx[0]][0] # NN in latent space P0_latent_list = list( map( lambda x: trainer.network.encode( x.transpose(1, 2).contiguous(), x.transpose(1, 2).contiguous()), points_train_list)) cosine_list = list( map(lambda x: loss.cosine(x, P2_latent), P0_latent_list)) top_k_idx, top_k_values = min_k(cosine_list) return_dict["cosine_criterion"] = point_train_paths[top_k_idx[0]][0] # Cycle 2 P0_P2_cycle_list = list( map(lambda x, y: loss.batch_cycle_2(x[0], y[3], 1), P0_P2_list, P2_P0_list)) P0_P2_cycle_list = list( map(lambda x, y: loss.L2(x, y), P0_P2_cycle_list, points_train_list)) P2_P0_cycle_list = list( map(lambda x, y: loss.batch_cycle_2(x[0], y[3], 1), P2_P0_list, P0_P2_list)) P2_P0_cycle_list = list(map(lambda x: loss.L2(x, P2), P2_P0_cycle_list)) # Cycle 2 both sides both_cycle_list = list( map(lambda x, y: x * y, P0_P2_cycle_list, P2_P0_cycle_list)) both_cycle_list = np.power(both_cycle_list, 1.0 / 2.0).tolist() top_k_cycle2_idx, top_k_values = min_k(both_cycle_list) return_dict["cycle_criterion"] = point_train_paths[ top_k_cycle2_idx[0]][0] pprint.pprint(return_dict) return return_dict
def forward(opt): """ Takes an input and a target mesh. Deform input in output and propagate a manually defined high frequency from the oinput to the output :return: """ my_utils.plant_seeds(randomized_seed=opt.randomize) os.makedirs(opt.output_dir, exist_ok=True) trainer = t.Trainer(opt) trainer.build_dataset_train_for_matching() trainer.build_dataset_test_for_matching() trainer.build_network() trainer.build_losses() trainer.network.eval() if opt.eval_list and os.path.isfile(opt.eval_list): source_target_files = np.loadtxt(opt.eval_list, dtype=str) source_target_files = source_target_files.tolist() for i, st in enumerate(source_target_files): source, target = st cat1, fname1 = source.split('/') fname1 = os.path.splitext(fname1)[0] cat2, fname2 = target.split('/') fname2 = os.path.splitext(fname2)[0] if len(opt.shapenetv1_path) > 0: source_target_files[i] = (os.path.join(opt.shapenetv1_path, cat1, fname1, "model.obj"), os.path.join(opt.shapenetv1_path, cat2, fname2, "model.obj")) elif len(opt.shapenetv2_path) > 0: source_target_files[i] = (os.path.join(opt.shapenetv2_path, cat1, fname1, "models", "model_normalized.obj"), os.path.join(opt.shapenetv2_path, cat2, fname2, "models", "model_normalized.obj")) elif (opt.eval_source != "" and opt.eval_source[-4:] == ".txt") and ( opt.eval_target != "" and opt.eval_target[-4:] == ".txt"): source_target_files = [ (figure_2_3.convert_path(opt.shapenetv1_path, opt.eval_source), figure_2_3.convert_path(opt.shapenetv1_path, opt.eval_target)) ] rot_mat = get_3D_rot_matrix(1, np.pi / 2) rot_mat_rev = get_3D_rot_matrix(1, -np.pi / 2) isV2 = len(opt.shapenetv2_path) > 0 for i, source_target in enumerate(source_target_files): basename = get_model_id(source_target[0], isV2) + "-" + get_model_id( source_target[1], isV2) path_deformed = os.path.join(opt.output_dir, basename + "-Sab.ply") path_source = os.path.join(opt.output_dir, basename + "-Sa.ply") path_target = os.path.join(opt.output_dir, basename + "-Sb.ply") mesh_path = source_target[0] print(mesh_path) source_mesh_edge = get_shapenet_model.link(mesh_path) mesh_path = source_target[1] target_mesh_edge = get_shapenet_model.link(mesh_path) print("Deforming source in target") source = source_mesh_edge.vertices target = target_mesh_edge.vertices pymesh.save_mesh_raw(path_source, source, source_mesh_edge.faces, ascii=True) pymesh.save_mesh_raw(path_target, target, target_mesh_edge.faces, ascii=True) if len(opt.shapenetv2_path) > 0: source = source.dot(rot_mat) target = target.dot(rot_mat) source = torch.from_numpy(source).cuda().float().unsqueeze(0) target = torch.from_numpy(target).cuda().float().unsqueeze(0) with torch.no_grad(): source, _, _, _, _ = loss.forward_chamfer( trainer.network, source, target, local_fix=None, distChamfer=trainer.distChamfer) try: source = source.squeeze().cpu().detach().numpy() if len(opt.shapenetv2_path) > 0: source = source.dot(rot_mat_rev) P2_P1_mesh = pymesh.form_mesh(vertices=source, faces=source_mesh_edge.faces) pymesh.save_mesh(path_deformed, P2_P1_mesh, ascii=True) # print("computing signal tranfer form source to target") # high_frequencies.high_frequency_propagation(path_source, path_deformed, path_target) except Exception as e: print(e) import pdb pdb.set_trace() path_deformed = path_deformed[:-4] + ".pts" save_pts(path_deformed, source.squeeze().cpu().detach().numpy())
# Nearest Neighbor (NN): # identity_nn = distChamfer(src_points.unsqueeze(0).cuda(), tgt_points.unsqueeze(0).cuda()) # identity_pred_labels = src_labels[identity_nn[3].view(-1).cpu().data.long()] # visualize_points(tgt_points.cpu().numpy(), bound=1.0, c=identity_pred_labels.cpu().numpy(), out_file=os.path.join(figures_folder, 'tgt_' + str(j) + '_nn.png')) # ICP + NN: # icp_src_points = ICP.ICP(src_points.unsqueeze(0), tgt_points.unsqueeze(0)) # visualize_points(icp_src_points.cpu().numpy(), bound=1.0, c=src_labels.cpu().numpy(), out_file=os.path.join(figures_folder, 'tgt_' + str(j) + '_icp_inter.png')) # icp_nn = distChamfer(icp_src_points.unsqueeze(0).cuda(), tgt_points.unsqueeze(0).cuda()) # icp_pred_labels = src_labels[icp_nn[3].view(-1).cpu().data.long()] # visualize_points(tgt_points.cpu().numpy(), bound=1.0, c=icp_pred_labels.cpu().numpy(), out_file=os.path.join(figures_folder, 'tgt_' + str(j) + '_icp.png')) # Cycle Consistency model: cc_forward = loss.forward_chamfer(trainer.network, src_points.unsqueeze(0).cuda(), tgt_points.unsqueeze(0).cuda(), local_fix=None, distChamfer=distChamfer) cc_src_points = cc_forward[0].squeeze(0) visualize_points(cc_src_points.cpu().numpy(), bound=1.0, c=src_labels.cpu().numpy(), out_file=os.path.join( figures_folder, 'tgt_' + str(j) + '_cc_inter.png'), show=True) cc_pred_labels = src_labels[cc_forward[4].view( -1).cpu().data.long()] visualize_points(tgt_points.cpu().numpy(), bound=1.0, c=cc_pred_labels,
def forward(opt): """ Takes an input and a target mesh. Deform input in output and propagate a manually defined high frequency from the oinput to the output :return: """ my_utils.plant_seeds(randomized_seed=opt.randomize) pdb.set_trace() trainer = t.Trainer(opt) trainer.build_dataset_train_for_matching() trainer.build_dataset_test_for_matching() trainer.build_network() trainer.build_losses() trainer.network.eval() if opt.eval_source[-4:] == ".txt": opt.eval_source = figure_2_3.convert_path(opt.shapenetv1_path, opt.eval_source) if opt.eval_target[-4:] == ".txt": opt.eval_target = figure_2_3.convert_path(opt.shapenetv1_path, opt.eval_target) path_deformed = os.path.join("./figures/forward_input_target/", opt.eval_source[-42:-10] + "deformed.ply") path_source = os.path.join("./figures/forward_input_target/", opt.eval_source[-42:-10] + ".ply") path_target = os.path.join( "./figures/forward_input_target/", opt.eval_source[-42:-10] + "_" + opt.eval_target[-42:-10] + ".ply") mesh_path = opt.eval_source print(mesh_path) source_mesh_edge = get_shapenet_model.link(mesh_path) mesh_path = opt.eval_target target_mesh_edge = get_shapenet_model.link(mesh_path) pymesh.save_mesh(path_source, source_mesh_edge, ascii=True) pymesh.save_mesh(path_target, target_mesh_edge, ascii=True) print("Deforming source in target") source = torch.from_numpy( source_mesh_edge.vertices).cuda().float().unsqueeze(0) target = torch.from_numpy( target_mesh_edge.vertices).cuda().float().unsqueeze(0) with torch.no_grad(): source, _, _, _, _ = loss.forward_chamfer( trainer.network, source, target, local_fix=None, distChamfer=trainer.distChamfer) P2_P1_mesh = pymesh.form_mesh( vertices=source.squeeze().cpu().detach().numpy(), faces=source_mesh_edge.faces) pymesh.save_mesh(path_deformed, P2_P1_mesh, ascii=True) print("computing signal tranfer form source to target") high_frequencies.high_frequency_propagation(path_source, path_deformed, path_target)
with torch.no_grad(): for i in tqdm.tqdm(range(trainer.len_dataset_test)): try: P1, _, _, _, P2, _, _, _ = trainer.dataset_test[i] # P2, _, _, _ = trainer.dataset_test[i] P1 = torch.tensor( P1[:, :3]).unsqueeze(0).contiguous().cuda().float() P2 = torch.tensor( P2[:, :3]).unsqueeze(0).contiguous().cuda().float() except: break visualize_points(P1[0].cpu().numpy(), bound=0.5, show=True) visualize_points(P2[0].cpu().numpy(), bound=0.5, show=True) # pdb.set_trace() P1_2 = loss.forward_chamfer(trainer.network, P1, P2, local_fix=None, distChamfer=distChamfer) visualize_points(P1_2[0][0].cpu().numpy(), bound=0.5, show=True) visualize_points_overlay( [P1_2[0][0].cpu().numpy(), P2[0].cpu().numpy()], bound=0.5, show=True)