def eval_final(): """ Evaluated model on test set in an extended way: computes estimates over multiple samples of point clouds and stores predictions """ model.eval() acc_meter = tnt.meter.ClassErrorMeter(accuracy=True) confusion_matrix = metrics.ConfusionMatrix(dbinfo['classes']) collected, predictions = defaultdict(list), {} # collect predictions over multiple sampling seeds for ss in range(args.test_multisamp_n): test_dataset_ss = create_dataset(args, ss)[1] loader = torch.utils.data.DataLoader(test_dataset_ss, batch_size=1, collate_fn=spg.eccpc_collate, num_workers=args.nworkers) if logging.getLogger().getEffectiveLevel() > logging.DEBUG: loader = tqdm(loader, ncols=65) # iterate over dataset in batches for bidx, (targets, GIs, clouds_data) in enumerate(loader): model.ecc.set_info(GIs, args.cuda) label_mode_cpu, label_vec_cpu, segm_size_cpu = targets[:, 0], targets[:, 2:], targets[:, 1:].sum( 1 ).float( ) embeddings = ptnCloudEmbedder.run(model, *clouds_data) outputs = model.ecc(embeddings) fname = clouds_data[0][0][:clouds_data[0][0].rfind('.')] collected[fname].append( (outputs.data.cpu().numpy(), label_mode_cpu.numpy(), label_vec_cpu.numpy())) # aggregate predictions (mean) for fname, lst in collected.items(): o_cpu, t_cpu, tvec_cpu = list(zip(*lst)) if args.test_multisamp_n > 1: o_cpu = np.mean(np.stack(o_cpu, 0), 0) else: o_cpu = o_cpu[0] t_cpu, tvec_cpu = t_cpu[0], tvec_cpu[0] predictions[fname] = np.argmax(o_cpu, 1) o_cpu, t_cpu, tvec_cpu = filter_valid(o_cpu, t_cpu, tvec_cpu) if t_cpu.size > 0: acc_meter.add(o_cpu, t_cpu) confusion_matrix.count_predicted_batch(tvec_cpu, np.argmax(o_cpu, 1)) per_class_iou = {} perclsiou = confusion_matrix.get_intersection_union_per_class() for c, name in dbinfo['inv_class_map'].items(): print('dbinfo: ', dbinfo['inv_class_map'], perclsiou[c]) per_class_iou[name] = perclsiou[c] return meter_value(acc_meter), confusion_matrix.get_overall_accuracy( ), confusion_matrix.get_average_intersection_union( ), per_class_iou, predictions, confusion_matrix.get_mean_class_accuracy( ), confusion_matrix.confusion_matrix
def eval(is_valid=False): """ Evaluated model on test set """ model.eval() if is_valid: #validation loader = torch.utils.data.DataLoader(valid_dataset, batch_size=1, collate_fn=spg.eccpc_collate, num_workers=args.nworkers) else: #evaluation loader = torch.utils.data.DataLoader(test_dataset, batch_size=1, collate_fn=spg.eccpc_collate, num_workers=args.nworkers) if logging.getLogger().getEffectiveLevel() > logging.DEBUG: loader = tqdm(loader, ncols=65) acc_meter = tnt.meter.ClassErrorMeter(accuracy=True) loss_meter = tnt.meter.AverageValueMeter() confusion_matrix = metrics.ConfusionMatrix(dbinfo['classes']) # iterate over dataset in batches for bidx, (targets, GIs, clouds_data) in enumerate(loader): model.ecc.set_info(GIs, args.cuda) label_mode_cpu, label_vec_cpu, segm_size_cpu = targets[:, 0], targets[:, 2:], targets[:, 1:].sum( 1 ).float( ) if args.cuda: label_mode, label_vec, segm_size = label_mode_cpu.cuda( ), label_vec_cpu.float().cuda(), segm_size_cpu.float().cuda() else: label_mode, label_vec, segm_size = label_mode_cpu, label_vec_cpu.float( ), segm_size_cpu.float() embeddings = ptnCloudEmbedder.run(model, *clouds_data) outputs = model.ecc(embeddings) loss = nn.functional.cross_entropy(outputs, Variable(label_mode), weight=dbinfo["class_weights"]) loss_meter.add(loss.item()) o_cpu, t_cpu, tvec_cpu = filter_valid(outputs.data.cpu().numpy(), label_mode_cpu.numpy(), label_vec_cpu.numpy()) if t_cpu.size > 0: acc_meter.add(o_cpu, t_cpu) confusion_matrix.count_predicted_batch(tvec_cpu, np.argmax(o_cpu, 1)) return meter_value(acc_meter), loss_meter.value( )[0], confusion_matrix.get_overall_accuracy( ), confusion_matrix.get_average_intersection_union( ), confusion_matrix.get_mean_class_accuracy()
def evaluate(i_epoch): """ Evaluated model on test set """ model.eval() with torch.no_grad(): loader = torch.utils.data.DataLoader(test_dataset, batch_size=1, collate_fn=graph_collate, num_workers=args.nworkers) if logging.getLogger().getEffectiveLevel() > logging.DEBUG: loader = tqdm(loader, ncols=100) loss_meter = tnt.meter.AverageValueMeter() n_clusters_meter = tnt.meter.AverageValueMeter() BR_meter = tnt.meter.AverageValueMeter() BP_meter = tnt.meter.AverageValueMeter() CM_classes = metrics.ConfusionMatrix(dbinfo['classes']) # iterate over dataset in batches for bidx, (fname, edg_source, edg_target, is_transition, labels, objects, clouds_data, xyz) in enumerate(loader): if args.cuda: is_transition = is_transition.to('cuda', non_blocking=True) # labels = torch.from_numpy(labels).cuda() objects = objects.to('cuda', non_blocking=True) clouds, clouds_global, nei = clouds_data clouds_data = (clouds.to('cuda', non_blocking=True), clouds_global.to('cuda', non_blocking=True), nei) embeddings = ptnCloudEmbedder.run_batch(model, *clouds_data, xyz) diff = compute_dist(embeddings, edg_source, edg_target, args.dist_type) if len(is_transition) > 1: weights_loss, pred_components, pred_in_component = compute_weight_loss(args, embeddings, objects, edg_source, edg_target, is_transition, diff, True, xyz) loss1, loss2 = compute_loss(args, diff, is_transition, weights_loss) loss = (loss1 + loss2) / weights_loss.shape[0] pred_transition = pred_in_component[edg_source] != pred_in_component[edg_target] per_pred = perfect_prediction(pred_components, labels) CM_classes.count_predicted_batch(labels[:, 1:], per_pred) else: loss = 0 if len(is_transition) > 1: loss_meter.add(loss.item()) # /weights_loss.sum().item()) is_transition = is_transition.cpu().numpy() n_clusters_meter.add(len(pred_components)) BR_meter.add((is_transition.sum()) * compute_boundary_recall(is_transition, relax_edge_binary(pred_transition, edg_source, edg_target, xyz.shape[0], args.BR_tolerance)), n=is_transition.sum()) BP_meter.add((pred_transition.sum()) * compute_boundary_precision(relax_edge_binary(is_transition, edg_source, edg_target, xyz.shape[0], args.BR_tolerance), pred_transition), n=pred_transition.sum()) CM = CM_classes.confusion_matrix return loss_meter.value()[0], n_clusters_meter.value()[0], 100 * CM.trace() / CM.sum(), BR_meter.value()[0], BP_meter.value()[0]
def evaluate_final(): """ Evaluated model on test set """ print("Final evaluation") model.eval() loss_meter = tnt.meter.AverageValueMeter() n_clusters_meter = tnt.meter.AverageValueMeter() confusion_matrix_classes = metrics.ConfusionMatrix(dbinfo['classes']) confusion_matrix_BR = metrics.ConfusionMatrix(2) confusion_matrix_BP = metrics.ConfusionMatrix(2) with torch.no_grad(): loader = torch.utils.data.DataLoader(test_dataset, batch_size=1, collate_fn=graph_collate, num_workers=args.nworkers) if logging.getLogger().getEffectiveLevel() > logging.DEBUG: loader = tqdm(loader, ncols=100) # iterate over dataset in batches for bidx, (fname, edg_source, edg_target, is_transition, labels, objects, clouds_data, xyz) in enumerate(loader): if args.cuda: is_transition = is_transition.to('cuda', non_blocking=True) # labels = torch.from_numpy(labels).cuda() objects = objects.to('cuda', non_blocking=True) clouds, clouds_global, nei = clouds_data clouds_data = (clouds.to('cuda', non_blocking=True), clouds_global.to('cuda', non_blocking=True), nei) if args.dataset == 'sema3d': embeddings = ptnCloudEmbedder.run_batch_cpu( model, *clouds_data, xyz) else: embeddings = ptnCloudEmbedder.run_batch( model, *clouds_data, xyz) diff = compute_dist(embeddings, edg_source, edg_target, args.dist_type) pred_components, pred_in_component = compute_partition( args, embeddings, edg_source, edg_target, diff, xyz) if len(is_transition) > 1: pred_transition = pred_in_component[ edg_source] != pred_in_component[edg_target] is_transition = is_transition.cpu().numpy() n_clusters_meter.add(len(pred_components)) per_pred = perfect_prediction(pred_components, labels) confusion_matrix_classes.count_predicted_batch( labels[:, 1:], per_pred) confusion_matrix_BR.count_predicted_batch_hard( is_transition, relax_edge_binary(pred_transition, edg_source, edg_target, xyz.shape[0], args.BR_tolerance).astype('uint8')) confusion_matrix_BP.count_predicted_batch_hard( relax_edge_binary(is_transition, edg_source, edg_target, xyz.shape[0], args.BR_tolerance), pred_transition.astype('uint8')) if args.spg_out: graph_sp = compute_sp_graph(xyz, 100, pred_in_component, pred_components, labels, dbinfo["classes"]) spg_file = os.path.join(folder_hierarchy.spg_folder, fname[0]) if not os.path.exists(os.path.dirname(spg_file)): os.makedirs(os.path.dirname(spg_file)) try: os.remove(spg_file) except OSError: pass write_spg(spg_file, graph_sp, pred_components, pred_in_component) # Debugging purpose - write the embedding file and an exemple of scalar files # if bidx % 0 == 0: # embedding2ply(os.path.join(folder_hierarchy.emb_folder , fname[0][:-3] + '_emb.ply'), xyz, embeddings.detach().cpu().numpy()) # scalar2ply(os.path.join(folder_hierarchy.scalars , fname[0][:-3] + '_elevation.ply') , xyz, clouds_data[1][:,1].cpu()) # edg_class = is_transition + 2*pred_transition # edge_class2ply2(os.path.join(folder_hierarchy.emb_folder , fname[0][:-3] + '_transition.ply'), edg_class, xyz, edg_source, edg_target) if len(is_transition) > 1: res_name = folder_hierarchy.outputdir + '/res.h5' res_file = h5py.File(res_name, 'w') res_file.create_dataset( 'confusion_matrix_classes', data=confusion_matrix_classes.confusion_matrix, dtype='uint64') res_file.create_dataset( 'confusion_matrix_BR', data=confusion_matrix_BR.confusion_matrix, dtype='uint64') res_file.create_dataset( 'confusion_matrix_BP', data=confusion_matrix_BP.confusion_matrix, dtype='uint64') res_file.create_dataset('n_clusters', data=n_clusters_meter.value()[0], dtype='uint64') res_file.close() return
def train(): """ Trains for one epoch """ model.train() loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, collate_fn=spg.eccpc_collate, num_workers=args.nworkers, shuffle=True, drop_last=True) if logging.getLogger().getEffectiveLevel() > logging.DEBUG: loader = tqdm(loader, ncols=65) loss_meter = tnt.meter.AverageValueMeter() acc_meter = tnt.meter.ClassErrorMeter(accuracy=True) confusion_matrix = metrics.ConfusionMatrix(dbinfo['classes']) t0 = time.time() # iterate over dataset in batches for bidx, (targets, GIs, clouds_data) in enumerate(loader): t_loader = 1000 * (time.time() - t0) model.ecc.set_info(GIs, args.cuda) label_mode_cpu, label_vec_cpu, segm_size_cpu = targets[:, 0], targets[:, 2:], targets[:, 1:].sum( 1 ) if args.cuda: label_mode, label_vec, segm_size = label_mode_cpu.cuda( ), label_vec_cpu.float().cuda(), segm_size_cpu.float().cuda() else: label_mode, label_vec, segm_size = label_mode_cpu, label_vec_cpu.float( ), segm_size_cpu.float() optimizer.zero_grad() t0 = time.time() embeddings = ptnCloudEmbedder.run(model, *clouds_data) outputs = model.ecc(embeddings) loss = nn.functional.cross_entropy(outputs, Variable(label_mode), weight=dbinfo["class_weights"]) loss.backward() ptnCloudEmbedder.bw_hook() if args.grad_clip > 0: for p in model.parameters(): p.grad.data.clamp_(-args.grad_clip, args.grad_clip) optimizer.step() t_trainer = 1000 * (time.time() - t0) #loss_meter.add(loss.data[0]) # pytorch 0.3 loss_meter.add(loss.item()) # pytorch 0.4 o_cpu, t_cpu, tvec_cpu = filter_valid(outputs.data.cpu().numpy(), label_mode_cpu.numpy(), label_vec_cpu.numpy()) acc_meter.add(o_cpu, t_cpu) confusion_matrix.count_predicted_batch(tvec_cpu, np.argmax(o_cpu, 1)) logging.debug( 'Batch loss %f, Loader time %f ms, Trainer time %f ms.', loss.data.item(), t_loader, t_trainer) t0 = time.time() return acc_meter.value()[0], loss_meter.value( )[0], confusion_matrix.get_overall_accuracy( ), confusion_matrix.get_average_intersection_union()