def test(modelin=args.model,outfile=args.out,optimize=args.opt,ft=args.ft): # define model, dataloader, 3dmm eigenvectors, optimization method calib_net = PointNet(n=1,feature_transform=ft) sfm_net = PointNet(n=199,feature_transform=ft) if modelin != "": calib_path = os.path.join('model','calib_' + modelin) sfm_path = os.path.join('model','sfm_' + modelin) calib_net.load_state_dict(torch.load(calib_path)) sfm_net.load_state_dict(torch.load(sfm_path)) calib_net.eval() sfm_net.eval() # mean shape and eigenvectors for 3dmm M = 100 data3dmm = dataloader.SyntheticLoader() mu_lm = torch.from_numpy(data3dmm.mu_lm).float().detach() mu_lm[:,2] = mu_lm[:,2]*-1 lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float().detach() sigma = torch.from_numpy(data3dmm.sigma).float().detach() sigma = torch.diag(sigma.squeeze()) lm_eigenvec = torch.mm(lm_eigenvec, sigma) # sample from f testing set allerror_2d = [] allerror_3d = [] allerror_rel3d = [] allerror_relf = [] all_f = [] all_fpred = [] all_depth = [] out_shape = [] out_f = [] seterror_3d = [] seterror_rel3d = [] seterror_relf = [] seterror_2d = [] f_vals = [i*100 for i in range(4,15)] # set random seed for reproducibility of test set np.random.seed(0) torch.manual_seed(0) for f_test in f_vals: # create dataloader loader = dataloader.TestLoader(f_test) f_pred = [] shape_pred = [] error_2d = [] error_3d = [] error_rel3d = [] error_relf = [] M = 100; N = 68; batch_size = 1; for j,data in enumerate(loader): if j >= 10: break # load the data x_cam_gt = data['x_cam_gt'] shape_gt = data['x_w_gt'] fgt = data['f_gt'] x_img = data['x_img'] x_img_gt = data['x_img_gt'] depth = torch.norm(x_cam_gt.mean(2),dim=1) all_depth.append(depth.numpy()) all_f.append(fgt.numpy()[0]) ptsI = x_img.reshape((M,N,2)).permute(0,2,1) x = x_img.unsqueeze(0).permute(0,2,1) # run the model f = calib_net(x) + 300 betas = sfm_net(x) betas = betas.squeeze(0).unsqueeze(-1) shape = mu_lm + torch.mm(lm_eigenvec,betas).squeeze().view(N,3) shape = shape - shape.mean(0).unsqueeze(0) # get motion measurement guess K = torch.zeros((3,3)).float() K[0,0] = f K[1,1] = f K[2,2] = 1 km,c_w,scaled_betas,alphas = util.EPnP(ptsI,shape,K) _, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI) error_time = util.getTimeConsistency(shape,R,T) if error_time > 10: mode='walk' else: mode='still' # apply dual optimization if optimize: calib_net.load_state_dict(torch.load(calib_path)) sfm_net.load_state_dict(torch.load(sfm_path)) shape,K,R,T = dualoptimization(x,calib_net,sfm_net,shape_gt=shape_gt,fgt=fgt,mode=mode) f = K[0,0].detach() else: K = torch.zeros(3,3).float() K[0,0] = f K[1,1] = f K[2,2] = 1 km,c_w,scaled_betas,alphas = util.EPnP(ptsI,shape,K) Xc, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI) # get errors reproj_errors2 = util.getReprojError2(ptsI,shape,R,T,K,show=False) reproj_errors3 = torch.norm(shape_gt - shape,dim=1).mean() rel_errors = util.getRelReprojError3(x_cam_gt,shape,R,T) reproj_error = reproj_errors2.mean() reconstruction_error = reproj_errors3.mean() rel_error = rel_errors.mean() f_error = torch.abs(fgt - f) / fgt # save final prediction f_pred.append(f.detach().cpu().item()) shape_pred.append(shape.detach().cpu().numpy()) all_fpred.append(f.detach().data.numpy()) allerror_3d.append(reproj_error.data.numpy()) allerror_2d.append(reconstruction_error.data.numpy()) allerror_rel3d.append(rel_error.data.numpy()) error_2d.append(reproj_error.cpu().data.item()) error_3d.append(reconstruction_error.cpu().data.item()) error_rel3d.append(rel_error.cpu().data.item()) error_relf.append(f_error.cpu().data.item()) print(f"f/sequence: {f_test}/{j} | f/fgt: {f.item():.3f}/{fgt.item():.3f} | f_error_rel: {f_error.item():.4f} | rmse: {reconstruction_error.item():.4f} | rel rmse: {rel_error.item():.4f} | 2d error: {reproj_error.item():.4f}") avg_2d = np.mean(error_2d) avg_rel3d = np.mean(error_rel3d) avg_3d = np.mean(error_3d) avg_relf = np.mean(error_relf) seterror_2d.append(avg_2d) seterror_3d.append(avg_3d) seterror_rel3d.append(avg_rel3d) seterror_relf.append(avg_relf) out_f.append(np.stack(f_pred)) out_shape.append(np.concatenate(shape_pred,axis=0)) print(f"f_error_rel: {avg_relf:.4f} | rel rmse: {avg_rel3d:.4f} | 2d error: {avg_2d:.4f} | rmse: {avg_3d:.4f} |") # save output out_shape = np.stack(out_shape) out_f = np.stack(out_f) all_f = np.stack(all_f).flatten() all_fpred = np.stack(all_fpred).flatten() all_d = np.stack(all_depth).flatten() allerror_2d = np.stack(allerror_2d).flatten() allerror_rel3d = np.stack(allerror_rel3d).flatten() matdata = {} matdata['fvals'] = np.array(f_vals) matdata['all_f'] = np.array(all_f) matdata['all_fpred'] = np.array(all_fpred) matdata['all_d'] = np.array(all_depth) matdata['error_2d'] = allerror_2d matdata['error_3d'] = allerror_3d matdata['error_rel3d'] = allerror_rel3d matdata['seterror_2d'] = np.array(seterror_2d) matdata['seterror_3d'] = np.array(seterror_3d) matdata['seterror_rel3d'] = np.array(seterror_rel3d) matdata['seterror_relf'] = np.array(seterror_relf) matdata['shape'] = np.stack(out_shape) matdata['f'] = np.stack(out_f) scipy.io.savemat(outfile,matdata) print(f"MEAN seterror_2d: {np.mean(seterror_2d)}") print(f"MEAN seterror_3d: {np.mean(seterror_3d)}") print(f"MEAN seterror_rel3d: {np.mean(seterror_rel3d)}") print(f"MEAN seterror_relf: {np.mean(seterror_relf)}") return np.mean(seterror_relf)
def dualoptimization(x,calib_net,sfm_net,shape_gt=None,fgt=None,M=100,N=68,mode='still',db='real'): if mode == 'still': alpha = 1 else: alpha = 0.0001 # define what weights gets optimized calib_net.eval() sfm_net.eval() trainfc(calib_net) trainfc(sfm_net) ptsI = x.squeeze().permute(1,0).reshape((M,N,2)).permute(0,2,1) # run the model f = calib_net(x) + 300 betas = sfm_net(x) betas = betas.squeeze(0).unsqueeze(-1) shape = mu_lm + torch.mm(lm_eigenvec,betas).squeeze().view(N,3) shape = shape - shape.mean(0).unsqueeze(0) opt1 = torch.optim.Adam(calib_net.parameters(),lr=1e-4) opt2 = torch.optim.Adam(sfm_net.parameters(),lr=5) curloss = 100 for outerloop in itertools.count(): shape = shape.detach() for iter in itertools.count(): opt1.zero_grad() f = calib_net(x) + 300 K = torch.zeros(3,3).float() K[0,0] = f K[1,1] = f K[2,2] = 1 # pose estimation km,c_w,scaled_betas, alphas = util.EPnP(ptsI,shape,K) _, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI) Xc = torch.bmm(R,torch.stack(M*[shape.T])) + T.unsqueeze(2) shape_error = util.getShapeError(ptsI,Xc,shape,f,R,T) error_time = util.getTimeConsistency(shape,R,T) error2d = util.getReprojError2(ptsI,shape,R,T,K,show=False,loss='l2') # apply loss #loss = shape_error loss = error2d.mean() + alpha*error_time #loss = error2d.mean() # database constraint if db == 'biwi': dgt = 1000 dpred = torch.norm(T,dim=1) loss = loss + 0.001*torch.abs(dgt - dpred).mean() if iter >= 5: break prv_loss = loss.item() loss.backward() opt1.step() # log results on console if not shape_gt is None: rmse = torch.norm(shape_gt - shape,dim=1).mean().item() else: rmse = -1 if not fgt is None: ftrue = fgt.item() else: fgt = -1 f_error = torch.mean(torch.abs(f-ftrue)) print(f"iter: {iter} | error: {loss.item():.3f} | f/fgt: {f.item():.1f}/{ftrue:.1f} | error2d: {error2d.mean().item():.3f} | error_time: {error_time.item():.2f} | rmse: {rmse:.2f}") f = f.detach() for iter in itertools.count(): opt2.zero_grad() # shape prediction betas = sfm_net(x) shape = torch.sum(betas * lm_eigenvec,1) shape = shape.reshape(68,3) + mu_lm shape = shape - shape.mean(0).unsqueeze(0) K = torch.zeros((3,3)).float() K[0,0] = f K[1,1] = f K[2,2] = 1 # differentiable PnP pose estimation km,c_w,scaled_betas,alphas = util.EPnP(ptsI,shape,K) _, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI) error2d = util.getReprojError2(ptsI,shape,R,T,K,show=False,loss='l2') Xc = torch.bmm(R,torch.stack(M*[shape.T])) + T.unsqueeze(2) shape_error = util.getShapeError(ptsI,Xc,shape,f,R,T) error_time = util.getTimeConsistency(shape,R,T) # apply loss #loss = shape_error loss = error2d.mean() + alpha*error_time #loss = error2d.mean() # database constraint if db == 'biwi': dgt = 1000 dpred = torch.norm(T,dim=1) loss = loss + 0.001*torch.abs(dgt - dpred).mean() if iter >= 5: break loss.backward() opt2.step() prv_loss = loss.item() # log results on console if not shape_gt is None: rmse = torch.norm(shape_gt - shape,dim=1).mean().item() else: rmse = -1 if not fgt is None: ftrue = fgt.item() else: fgt = -1 f_error = torch.mean(torch.abs(f-ftrue)) print(f"iter: {iter} | error: {loss.item():.3f} | f/fgt: {f.item():.1f}/{ftrue:.1f} | error2d: {error2d.mean().item():.3f} | error_time: {error_time.item():.2f} | rmse: {rmse:.2f}") if torch.abs(curloss - loss) <= 0.01 or curloss < loss: break curloss = loss return shape,K,R,T
def dualoptimization(x, calib_net, sfm_net, shape_gt=None, fgt=None, M=100, N=68, mode='still', ptstart=0): alphaModel = PointNetSmall(n=1) for module in alphaModel.modules(): if isinstance(module, torch.nn.modules.BatchNorm1d): module.eval() if isinstance(module, torch.nn.modules.BatchNorm2d): module.eval() if isinstance(module, torch.nn.modules.BatchNorm3d): module.eval() # define what weights gets optimized calib_net.eval() sfm_net.eval() trainfc(calib_net) trainfc(sfm_net) ptsI = x.squeeze().permute(1, 0).reshape((M, N, 2)).permute(0, 2, 1) ptsI = ptsI[:, :, ptstart:] # run the model f = calib_net(x) + 300 betas = sfm_net(x) betas = betas.squeeze(0).unsqueeze(-1) shape = mu_lm + torch.mm(lm_eigenvec, betas).squeeze().view(N, 3) shape = shape - shape.mean(0).unsqueeze(0) shape = shape[ptstart:, :] opt1 = torch.optim.Adam(list(calib_net.parameters()) + list(alphaModel.parameters()), lr=1e-5) opt2 = torch.optim.Adam(list(sfm_net.parameters()) + list(alphaModel.parameters()), lr=1) curloss = 100 for outerloop in itertools.count(): shape = shape.detach() for iter in itertools.count(): opt1.zero_grad() f = calib_net(x) + 300 K = torch.zeros(3, 3).float() K[0, 0] = f K[1, 1] = f K[2, 2] = 1 alpha = alphaModel(x) # pose estimation km, c_w, scaled_betas, alphas = util.EPnP(ptsI, shape, K) _, R, T, mask = util.optimizeGN(km, c_w, scaled_betas, alphas, shape, ptsI, K) Xc = torch.bmm(R, torch.stack(M * [shape.T])) + T.unsqueeze(2) shape_error = util.getShapeError(ptsI, Xc, shape, f, R, T) error_time = util.getTimeConsistency(shape, R, T) error2d = util.getReprojError2(ptsI, shape, R, T, K, show=False, loss='l2') # apply loss loss = error2d.mean() + alpha * error_time - torch.log(alpha) if iter >= 5: break prv_loss = loss.item() loss.backward() opt1.step() # log results on console if not shape_gt is None: rmse = torch.norm(shape_gt - shape, dim=1).mean().item() else: rmse = -1 if not fgt is None: ftrue = fgt.item() else: fgt = -1 f_error = torch.mean(torch.abs(f - ftrue)) print( f"iter: {iter} | error: {loss.item():.3f} | f/fgt: {f.item():.1f}/{ftrue:.1f} | error2d: {error2d.mean().item():.3f} | shape_error: {shape_error.item():.3f} | rmse: {rmse:.2f} | alpha: {alpha.item():.4f} | error_time: {error_time:.2f}" ) f = f.detach() for iter in itertools.count(): opt2.zero_grad() # shape prediction betas = sfm_net(x) shape = torch.sum(betas * lm_eigenvec, 1) shape = shape.reshape(68, 3) + mu_lm shape = shape - shape.mean(0).unsqueeze(0) shape = shape[ptstart:, :] K = torch.zeros((3, 3)).float() K[0, 0] = f K[1, 1] = f K[2, 2] = 1 alpha = alphaModel(x) # differentiable PnP pose estimation km, c_w, scaled_betas, alphas = util.EPnP(ptsI, shape, K) _, R, T, mask = util.optimizeGN(km, c_w, scaled_betas, alphas, shape, ptsI, K) error2d = util.getReprojError2(ptsI, shape, R, T, K, show=False, loss='l2') Xc = torch.bmm(R, torch.stack(M * [shape.T])) + T.unsqueeze(2) shape_error = util.getShapeError(ptsI, Xc, shape, f, R, T) error_time = util.getTimeConsistency(shape, R, T) # apply loss #loss = error2d.mean() loss = error2d.mean() + alpha * error_time - torch.log(alpha) #if iter >= 5 and loss > prv_loss: break if iter >= 5: break loss.backward() opt2.step() prv_loss = loss.item() # log results on console if not shape_gt is None: rmse = torch.norm(shape_gt - shape, dim=1).mean().item() else: rmse = -1 if not fgt is None: ftrue = fgt.item() else: fgt = -1 f_error = torch.mean(torch.abs(f - ftrue)) print( f"iter: {iter} | error: {loss.item():.3f} | f/fgt: {f.item():.1f}/{ftrue:.1f} | error2d: {error2d.mean().item():.3f} | rmse: {rmse:.2f} | alpha: {alpha.item():.4f} | error_time: {error_time:.2f}" ) if torch.abs(curloss - loss) <= 0.01 or curloss < loss: break curloss = loss return shape, K, R, T
def testReal(modelin=args.model,outfile=args.out,optimize=args.opt,db=args.db): # define model, dataloader, 3dmm eigenvectors, optimization method calib_net = PointNet(n=1) sfm_net = PointNet(n=199) if modelin != "": calib_path = os.path.join('model','calib_' + modelin) sfm_path = os.path.join('model','sfm_' + modelin) calib_net.load_state_dict(torch.load(calib_path)) sfm_net.load_state_dict(torch.load(sfm_path)) calib_net.eval() sfm_net.eval() # mean shape and eigenvectors for 3dmm data3dmm = dataloader.SyntheticLoader() mu_lm = torch.from_numpy(data3dmm.mu_lm).float().detach() mu_lm[:,2] = mu_lm[:,2]*-1 lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float().detach() sigma = torch.from_numpy(data3dmm.sigma).float().detach() sigma = torch.diag(sigma.squeeze()) lm_eigenvec = torch.mm(lm_eigenvec, sigma) # define loader loader = getLoader(db) f_pred = [] shape_pred = [] error_2d = [] error_relf = [] error_rel3d = [] for sub in range(len(loader)): batch = loader[sub] x_cam_gt = batch['x_cam_gt'] fgt = batch['f_gt'] x_img = batch['x_img'] x_img_gt = batch['x_img_gt'] M = x_img_gt.shape[0] N = x_img_gt.shape[-1] ptsI = x_img.reshape((M,N,2)).permute(0,2,1) x = x_img.unsqueeze(0).permute(0,2,1) # run the model f = calib_net(x) + 300 betas = sfm_net(x) betas = betas.squeeze(0).unsqueeze(-1) shape = mu_lm + torch.mm(lm_eigenvec,betas).squeeze().view(N,3) shape = shape - shape.mean(0).unsqueeze(0) # get motion measurement guess K = torch.zeros((3,3)).float() K[0,0] = f K[1,1] = f K[2,2] = 1 km,c_w,scaled_betas,alphas = util.EPnP(ptsI,shape,K) _, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI) error_time = util.getTimeConsistency(shape,R,T) if error_time > 20: mode='walk' else: mode='still' # adjust number of landmarks M = x_img_gt.shape[0] N = x_img_gt.shape[-1] # additional optimization on initial solution if optimize: calib_net.load_state_dict(torch.load(calib_path)) sfm_net.load_state_dict(torch.load(sfm_path)) if db == 'biwi': shape_gt = batch['x_w_gt'] shape,K,R,T = dualoptimization(x,calib_net,sfm_net,shape_gt=shape_gt,fgt=fgt,M=M,N=N,mode=mode,db='biwi') else: shape,K,R,T = dualoptimization(x,calib_net,sfm_net,fgt=fgt,M=M,N=N,mode=mode) f = K[0,0].detach() else: K = torch.zeros(3,3).float() K[0,0] = f K[1,1] = f K[2,2] = 1 km,c_w,scaled_betas,alphas = util.EPnP(ptsI,shape,K) Xc, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI) # get errors reproj_errors2 = util.getReprojError2(ptsI,shape,R,T,K) rel_errors = util.getRelReprojError3(x_cam_gt,shape,R,T) reproj_error = reproj_errors2.mean() rel_error = rel_errors.mean() f_error = torch.abs(fgt - f) / fgt # save final prediction f_pred.append(f.detach().cpu().item()) shape_pred.append(shape.detach().cpu().numpy()) error_2d.append(reproj_error.cpu().data.item()) error_rel3d.append(rel_error.cpu().data.item()) error_relf.append(f_error.cpu().data.item()) print(f" f/fgt: {f.item():.3f}/{fgt.item():.3f} | f_error_rel: {f_error.item():.4f} | rel rmse: {rel_error.item():.4f} | 2d error: {reproj_error.item():.4f}") #end for # prepare output file out_shape = np.stack(shape_pred) out_f = np.stack(f_pred) matdata = {} matdata['shape'] = np.stack(out_shape) matdata['f'] = np.stack(out_f) matdata['error_2d'] = np.array(error_2d) matdata['error_rel3d'] = np.array(error_rel3d) matdata['error_relf'] = np.array(error_relf) scipy.io.savemat(outfile,matdata) print(f"MEAN seterror_2d: {np.mean(error_2d)}") print(f"MEAN seterror_rel3d: {np.mean(error_rel3d)}") print(f"MEAN seterror_relf: {np.mean(error_relf)}")
def dualoptimization(x,calib_net,sfm_net,shape_gt=None,fgt=None,M=100,N=68): # define what weights gets optimized calib_net.eval() sfm_net.eval() trainfc(calib_net) trainfc(sfm_net) ptsI = x.squeeze().permute(1,0).reshape((M,N,2)).permute(0,2,1) ptsI = ptsI[:,:,ptstart:] # run the model f = calib_net(x) + 300 betas = sfm_net(x) betas = betas.squeeze(0).unsqueeze(-1) shape = mu_lm + torch.mm(lm_eigenvec,betas).squeeze().view(N,3) shape = shape - shape.mean(0).unsqueeze(0) shape = shape[ptstart:,:] opt1 = torch.optim.Adam(calib_net.parameters(),lr=1e-4) opt2 = torch.optim.Adam(sfm_net.parameters(),lr=10) curloss = 100 for outerloop in itertools.count(): f = f.detach() for iter in itertools.count(): opt2.zero_grad() # shape prediction betas = sfm_net(x) shape = torch.sum(betas * lm_eigenvec,1) shape = shape.reshape(68,3) + mu_lm shape = shape - shape.mean(0).unsqueeze(0) shape = shape[ptstart:,:] K = torch.zeros((3,3)).float() K[0,0] = f K[1,1] = f K[2,2] = 1 # differentiable PnP pose estimation km,c_w,scaled_betas,alphas = util.EPnP(ptsI,shape,K) Xc, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI,K) error2d = util.getReprojError2(ptsI,shape,R,T,K,show=False,loss='l2') error_time = util.getTimeConsistency(shape,R,T) # apply loss loss = error2d.mean() + 0.01*error_time if iter >= 5 and loss > prv_loss: break loss.backward() opt2.step() prv_loss = loss.item() # log results on console if not shape_gt is None: rmse = torch.norm(shape_gt - shape,dim=1).mean().item() else: rmse = -1 if not fgt is None: ftrue = fgt.item() else: fgt = -1 f_error = torch.mean(torch.abs(f-ftrue)) print(f"iter: {iter} | error: {loss.item():.3f} | f/fgt: {f.item():.1f}/{ftrue:.1f} | error2d: {error2d.mean().item():.3f} | rmse: {rmse:.2f}") shape = shape.detach() for iter in itertools.count(): opt1.zero_grad() f = calib_net(x) + 300 K = torch.zeros(3,3).float() K[0,0] = f K[1,1] = f K[2,2] = 1 # pose estimation km,c_w,scaled_betas, alphas = util.EPnP(ptsI,shape,K) Xc, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI,K) error2d = util.getReprojError2(ptsI,shape,R,T,K,show=False,loss='l1') error_time = util.getTimeConsistency(shape,R,T) # apply loss loss = error2d.mean() + 0.01*error_time if iter >= 5 and loss > prv_loss: break prv_loss = loss.item() loss.backward() opt1.step() # log results on console if not shape_gt is None: rmse = torch.norm(shape_gt - shape,dim=1).mean().item() else: rmse = -1 if not fgt is None: ftrue = fgt.item() else: fgt = -1 f_error = torch.mean(torch.abs(f-ftrue)) print(f"iter: {iter} | error: {loss.item():.3f} | f/fgt: {f.item():.1f}/{ftrue:.1f} | error2d: {error2d.mean().item():.3f} | rmse: {rmse:.2f}") if torch.abs(curloss - loss) <= 0.01 or curloss < loss: break curloss = loss return shape,K,R,T
def test_sfm(modelin=args.model, outfile=args.out, optimize=args.opt): # define model, dataloader, 3dmm eigenvectors, optimization method calib_net = CalibrationNet3(n=1) sfm_net = CalibrationNet3(n=199) calib_path = os.path.join('model', 'calib_' + modelin) sfm_path = os.path.join('model', 'sfm_' + modelin) # mean shape and eigenvectors for 3dmm M = 100 data3dmm = dataloader.SyntheticLoader() mu_lm = torch.from_numpy(data3dmm.mu_lm).float().detach() mu_lm[:, 2] = mu_lm[:, 2] * -1 lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float().detach() sigma = torch.from_numpy(data3dmm.sigma).float().detach() sigma = torch.diag(sigma.squeeze()) lm_eigenvec = torch.mm(lm_eigenvec, sigma) # sample from f testing set allerror_2d = [] allerror_3d = [] allerror_rel3d = [] allerror_relf = [] all_f = [] all_fpred = [] all_depth = [] out_shape = [] out_f = [] seterror_3d = [] seterror_rel3d = [] seterror_relf = [] seterror_2d = [] f_vals = [i * 100 for i in range(4, 15)] for f_test in f_vals: # create dataloader #f_test = 1000 loader = dataloader.TestLoader(f_test) f_pred = [] shape_pred = [] error_2d = [] error_3d = [] error_rel3d = [] error_relf = [] M = 100 N = 68 batch_size = 1 training_pred = np.zeros((10, 100, 68, 3)) training_gt = np.zeros((10, 100, 68, 3)) for j, data in enumerate(loader): if j == 10: break # load the data x_cam_gt = data['x_cam_gt'] shape_gt = data['x_w_gt'] fgt = data['f_gt'] x_img = data['x_img'] x_img_gt = data['x_img_gt'] T_gt = data['T_gt'] all_depth.append(np.mean(T_gt[:, 2])) all_f.append(fgt.numpy()[0]) ptsI = x_img.reshape((M, N, 2)).permute(0, 2, 1) x = ptsI.unsqueeze(0).permute(0, 2, 1, 3) # test camera calibration #calib_net.load_state_dict(torch.load(calib_path)) opt2 = torch.optim.Adam(sfm_net.parameters(), lr=1e-5) sfm_net.eval() trainfc(sfm_net) f = 2000 for iter in itertools.count(): opt2.zero_grad() # shape prediction betas = sfm_net.forward2(x) betas = torch.clamp(betas, -20, 20) shape = torch.sum(betas * lm_eigenvec, 1) shape = shape.reshape(68, 3) + mu_lm shape = shape - shape.mean(0).unsqueeze(0) rmse = torch.norm(shape_gt - shape, dim=1).mean().detach() K = torch.zeros((3, 3)).float() K[0, 0] = f K[1, 1] = f K[2, 2] = 1 km, c_w, scaled_betas, alphas = util.EPnP(ptsI, shape, K) Xc, R, T, mask = util.optimizeGN(km, c_w, scaled_betas, alphas, shape, ptsI, K) error2d = util.getReprojError2(ptsI, shape, R, T, K, show=False, loss='l2') error_time = util.getTimeConsistency(shape, R, T) loss = error2d.mean() + 0.01 * error_time loss.backward() opt2.step() print( f"iter: {iter} | error: {loss.item():.3f} | error2d: {error2d.mean().item():.3f} | rmse: {rmse.item():.3f} " ) if iter == 100: break training_pred[j, iter, :, :] = shape.detach().cpu().numpy() training_gt[j, iter, :, :] = shape_gt.detach().cpu().numpy() # get errors reproj_errors2 = util.getReprojError2(ptsI, shape, R, T, K, show=False) reproj_errors3 = torch.norm(shape_gt - shape, dim=1).mean() rel_errors = util.getRelReprojError3(x_cam_gt, shape, R, T) reproj_error = reproj_errors2.mean() reconstruction_error = reproj_errors3.mean() rel_error = rel_errors.mean() f_error = torch.abs(fgt - f) / fgt # save final prediction shape_pred.append(shape.detach().cpu().numpy()) allerror_3d.append(reproj_error.data.numpy()) allerror_2d.append(reconstruction_error.data.numpy()) allerror_rel3d.append(rel_error.data.numpy()) error_2d.append(reproj_error.cpu().data.item()) error_3d.append(reconstruction_error.cpu().data.item()) error_rel3d.append(rel_error.cpu().data.item()) error_relf.append(f_error.cpu().data.item()) print( f"f/sequence: {f_test}/{j} | f/fgt: {f:.3f}/{fgt.item():.3f} | f_error_rel: {f_error.item():.4f} | rmse: {reconstruction_error.item():.4f} | rel rmse: {rel_error.item():.4f} | 2d error: {reproj_error.item():.4f}" ) avg_2d = np.mean(error_2d) avg_rel3d = np.mean(error_rel3d) avg_3d = np.mean(error_3d) avg_relf = np.mean(error_relf) seterror_2d.append(avg_2d) seterror_3d.append(avg_3d) seterror_rel3d.append(avg_rel3d) seterror_relf.append(avg_relf) out_shape.append(np.stack(shape_pred, axis=0)) print( f"f_error_rel: {avg_relf:.4f} | rel rmse: {avg_rel3d:.4f} | 2d error: {reproj_error.item():.4f} | rmse: {avg_3d:.4f} |" ) all_f = np.stack(all_f).flatten() all_d = np.stack(all_depth).flatten() allerror_2d = np.stack(allerror_2d).flatten() allerror_3d = np.stack(allerror_3d).flatten() allerror_rel3d = np.stack(allerror_rel3d).flatten() matdata = {} matdata['training_pred'] = training_pred matdata['training_gt'] = training_gt matdata['fvals'] = np.array(f_vals) matdata['all_f'] = np.array(all_f) matdata['all_d'] = np.array(all_depth) matdata['error_2d'] = allerror_2d matdata['error_3d'] = allerror_3d matdata['error_rel3d'] = allerror_rel3d matdata['seterror_2d'] = np.array(seterror_2d) matdata['seterror_3d'] = np.array(seterror_3d) matdata['seterror_rel3d'] = np.array(seterror_rel3d) matdata['seterror_relf'] = np.array(seterror_relf) scipy.io.savemat(outfile, matdata) print(f"MEAN seterror_2d: {np.mean(seterror_2d)}") print(f"MEAN seterror_3d: {np.mean(seterror_3d)}") print(f"MEAN seterror_rel3d: {np.mean(seterror_rel3d)}") print(f"MEAN seterror_relf: {np.mean(seterror_relf)}")
def testBIWIID(modelin=args.model,outfile=args.out,optimize=args.opt): # define model, dataloader, 3dmm eigenvectors, optimization method calib_net = CalibrationNet3(n=1) sfm_net = CalibrationNet3(n=199) if modelin != "": calib_path = os.path.join('model','calib_' + modelin) sfm_path = os.path.join('model','sfm_' + modelin) calib_net.load_state_dict(torch.load(calib_path)) sfm_net.load_state_dict(torch.load(sfm_path)) calib_net.eval() sfm_net.eval() # mean shape and eigenvectors for 3dmm data3dmm = dataloader.SyntheticLoader() mu_lm = torch.from_numpy(data3dmm.mu_lm).float().detach() mu_lm[:,2] = mu_lm[:,2]*-1 lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float().detach() sigma = torch.from_numpy(data3dmm.sigma).float().detach() sigma = torch.diag(sigma.squeeze()) lm_eigenvec = torch.mm(lm_eigenvec, sigma) # define loader loader = dataloader.BIWIIDLoader() f_pred = [] shape_pred = [] error_2d = [] error_relf = [] error_rel3d = [] for idx in range(len(loader)): batch = loader[idx] x_cam_gt = batch['x_cam_gt'] fgt = batch['f_gt'] x_img = batch['x_img'] x_img_gt = batch['x_img_gt'] M = x_img_gt.shape[0] N = 68 ptsI = x_img.reshape((M,N,2)).permute(0,2,1) x = ptsI.unsqueeze(0).permute(0,2,1,3) # run the model f = calib_net(x) + 300 betas = sfm_net(x) betas = betas.squeeze(0).unsqueeze(-1) shape = mu_lm + torch.mm(lm_eigenvec,betas).squeeze().view(N,3) # additional optimization on initial solution if optimize: calib_net.load_state_dict(torch.load(calib_path)) sfm_net.load_state_dict(torch.load(sfm_path)) calib_net.eval() sfm_net.eval() trainfc(calib_net) trainfc(sfm_net) opt1 = torch.optim.Adam(calib_net.parameters(),lr=1e-4) opt2 = torch.optim.Adam(sfm_net.parameters(),lr=1e-5) curloss = 100 for outerloop in itertools.count(): # camera calibration shape = shape.detach() for iter in itertools.count(): opt1.zero_grad() f = calib_net.forward2(x) + 300 K = torch.zeros(3,3).float() K[0,0] = f K[1,1] = f K[2,2] = 1 f_error = torch.mean(torch.abs(f - fgt)) #rmse = torch.norm(shape_gt - shape,dim=1).mean() # differentiable PnP pose estimation km,c_w,scaled_betas, alphas = util.EPnP(ptsI,shape,K) Xc, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI,K) error2d = util.getReprojError2(ptsI,shape,R,T,K,show=False,loss='l2') error_time = util.getTimeConsistency(shape,R,T) loss = error2d.mean() + 0.01*error_time if iter == 5: break #if iter > 10 and prev_loss < loss: # break #else: # prev_loss = loss loss.backward() opt1.step() print(f"iter: {iter} | error: {loss.item():.3f} | f/fgt: {f.item():.1f}/{fgt[0].item():.1f} | error2d: {error2d.mean().item():.3f} ") # sfm f = f.detach() for iter in itertools.count(): opt2.zero_grad() # shape prediction betas = sfm_net.forward2(x) shape = torch.sum(betas * lm_eigenvec,1) shape = shape.reshape(68,3) + mu_lm shape = shape - shape.mean(0).unsqueeze(0) K = torch.zeros((3,3)).float() K[0,0] = f K[1,1] = f K[2,2] = 1 #rmse = torch.norm(shape_gt - shape,dim=1).mean().detach() #rmse = torch.norm(shape_gt - shape,dim=1).mean().detach() # differentiable PnP pose estimation km,c_w,scaled_betas,alphas = util.EPnP(ptsI,shape,K) Xc, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI,K) error2d = util.getReprojError2(ptsI,shape,R,T,K,show=False,loss='l2') error_time = util.getTimeConsistency(shape,R,T) loss = error2d.mean() + 0.01*error_time if iter == 5: break prev_loss = loss.item() loss.backward() opt2.step() print(f"iter: {iter} | error: {loss.item():.3f} | f/fgt: {f.item():.1f}/{fgt[0].item():.1f} | error2d: {error2d.mean().item():.3f} ") # closing condition for outerloop on dual objective if torch.abs(curloss - loss) < 0.01: break curloss = loss else: K = torch.zeros(3,3).float() K[0,0] = f K[1,1] = f K[2,2] = 1 km,c_w,scaled_betas,alphas = util.EPnP(ptsI,shape,K) Xc, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI,K) # get errors reproj_errors2 = util.getReprojError2(ptsI,shape,R,T,K) rel_errors = util.getRelReprojError3(x_cam_gt,shape,R,T) reproj_error = reproj_errors2.mean() rel_error = rel_errors.mean() f_error = torch.abs(fgt - f) / fgt # save final prediction f_pred.append(f.detach().cpu().item()) shape_pred.append(shape.detach().cpu().numpy()) error_2d.append(reproj_error.cpu().data.item()) error_rel3d.append(rel_error.cpu().data.item()) error_relf.append(f_error.cpu().data.item()) print(f" f/fgt: {f[0].item():.3f}/{fgt.item():.3f} | f_error_rel: {f_error.item():.4f} | rel rmse: {rel_error.item():.4f} | 2d error: {reproj_error.item():.4f}") #end for # prepare output file out_shape = np.stack(shape_pred) out_f = np.stack(f_pred) matdata = {} matdata['shape'] = np.stack(out_shape) matdata['f'] = np.stack(out_f) matdata['error_2d'] = np.array(error_2d) matdata['error_rel3d'] = np.array(error_rel3d) matdata['error_relf'] = np.array(error_relf) scipy.io.savemat(outfile,matdata) print(f"MEAN seterror_2d: {np.mean(error_2d)}") print(f"MEAN seterror_rel3d: {np.mean(error_rel3d)}") print(f"MEAN seterror_relf: {np.mean(error_relf)}")
def test(modelin=args.model,outfile=args.out,optimize=args.opt): # define model, dataloader, 3dmm eigenvectors, optimization method calib_net = CalibrationNet3(n=1) sfm_net = CalibrationNet3(n=199) if modelin != "": calib_path = os.path.join('model','calib_' + modelin) sfm_path = os.path.join('model','sfm_' + modelin) calib_net.load_state_dict(torch.load(calib_path)) sfm_net.load_state_dict(torch.load(sfm_path)) calib_net.eval() sfm_net.eval() # mean shape and eigenvectors for 3dmm M = 100 data3dmm = dataloader.SyntheticLoader() mu_lm = torch.from_numpy(data3dmm.mu_lm).float().detach() mu_lm[:,2] = mu_lm[:,2]*-1 lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float().detach() sigma = torch.from_numpy(data3dmm.sigma).float().detach() sigma = torch.diag(sigma.squeeze()) lm_eigenvec = torch.mm(lm_eigenvec, sigma) # sample from f testing set allerror_2d = [] allerror_3d = [] allerror_rel3d = [] allerror_relf = [] all_f = [] all_fpred = [] all_depth = [] out_shape = [] out_f = [] seterror_3d = [] seterror_rel3d = [] seterror_relf = [] seterror_2d = [] f_vals = [i*100 for i in range(4,15)] for f_test in f_vals: # create dataloader #f_test = 1000 loader = dataloader.TestLoader(f_test) f_pred = [] shape_pred = [] error_2d = [] error_3d = [] error_rel3d = [] error_relf = [] M = 100; N = 68; batch_size = 1; for j,data in enumerate(loader): if j == 10: break # load the data x_cam_gt = data['x_cam_gt'] shape_gt = data['x_w_gt'] fgt = data['f_gt'] x_img = data['x_img'] x_img_gt = data['x_img_gt'] T_gt = data['T_gt'] all_depth.append(np.mean(T_gt[:,2])) all_f.append(fgt.numpy()[0]) ptsI = x_img.reshape((M,N,2)).permute(0,2,1) x = ptsI.unsqueeze(0).permute(0,2,1,3) # run the model f = calib_net(x) + 300 betas = sfm_net(x) betas = betas.squeeze(0).unsqueeze(-1) shape = mu_lm + torch.mm(lm_eigenvec,betas).squeeze().view(N,3) # additional optimization on initial solution if optimize: calib_net.load_state_dict(torch.load(calib_path)) sfm_net.load_state_dict(torch.load(sfm_path)) calib_net.eval() sfm_net.eval() trainfc(calib_net) trainfc(sfm_net) opt1 = torch.optim.Adam(calib_net.parameters(),lr=1e-4) opt2 = torch.optim.Adam(sfm_net.parameters(),lr=1e-2) curloss = 100 for outerloop in itertools.count(): # camera calibration shape = shape.detach() for iter in itertools.count(): opt1.zero_grad() f = calib_net.forward2(x) + 300 K = torch.zeros(3,3).float() K[0,0] = f K[1,1] = f K[2,2] = 1 f_error = torch.mean(torch.abs(f - fgt)) rmse = torch.norm(shape_gt - shape,dim=1).mean() # differentiable PnP pose estimation km,c_w,scaled_betas, alphas = util.EPnP(ptsI,shape,K) Xc, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI,K) error2d = util.getReprojError2(ptsI,shape,R,T,K,show=False,loss='l2') #error2d = util.getReprojError2_(ptsI,Xc,K,show=True,loss='l2') error_time = util.getTimeConsistency(shape,R,T) loss = error2d.mean() + 0.01*error_time if iter == 5: break loss.backward() opt1.step() print(f"iter: {iter} | error: {loss.item():.3f} | f/fgt: {f.item():.1f}/{fgt[0].item():.1f} | error2d: {error2d.mean().item():.3f} | rmse: {rmse.item():.3f} ") # sfm f = f.detach() for iter in itertools.count(): opt2.zero_grad() # shape prediction betas = sfm_net.forward2(x) shape = torch.sum(betas * lm_eigenvec,1) shape = shape.reshape(68,3) + mu_lm shape = shape - shape.mean(0).unsqueeze(0) K = torch.zeros((3,3)).float() K[0,0] = f K[1,1] = f K[2,2] = 1 #rmse = torch.norm(shape_gt - shape,dim=1).mean().detach() rmse = torch.norm(shape_gt - shape,dim=1).mean().detach() # differentiable PnP pose estimation km,c_w,scaled_betas,alphas = util.EPnP(ptsI,shape,K) Xc, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI,K) error2d = util.getReprojError2(ptsI,shape,R,T,K,show=False,loss='l2') error_time = util.getTimeConsistency(shape,R,T) loss = error2d.mean() + 0.01*error_time if iter == 5: break if iter > 10 and prev_loss < loss: break else: prev_loss = loss loss.backward() opt2.step() print(f"iter: {iter} | error: {loss.item():.3f} | f/fgt: {f.item():.1f}/{fgt[0].item():.1f} | error2d: {error2d.mean().item():.3f} | rmse: {rmse.item():.3f} ") # closing condition for outerloop on dual objective if torch.abs(curloss - loss) < 0.01: break curloss = loss else: K = torch.zeros(3,3).float() K[0,0] = f K[1,1] = f K[2,2] = 1 km,c_w,scaled_betas,alphas = util.EPnP(ptsI,shape,K) Xc, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI,K) all_fpred.append(f.detach().numpy()[0]) # get errors reproj_errors2 = util.getReprojError2(ptsI,shape,R,T,K,show=False) reproj_errors3 = torch.norm(shape_gt - shape,dim=1).mean() rel_errors = util.getRelReprojError3(x_cam_gt,shape,R,T) reproj_error = reproj_errors2.mean() reconstruction_error = reproj_errors3.mean() rel_error = rel_errors.mean() f_error = torch.abs(fgt - f) / fgt # save final prediction f_pred.append(f.detach().cpu().item()) shape_pred.append(shape.detach().cpu().numpy()) allerror_3d.append(reproj_error.data.numpy()) allerror_2d.append(reconstruction_error.data.numpy()) allerror_rel3d.append(rel_error.data.numpy()) error_2d.append(reproj_error.cpu().data.item()) error_3d.append(reconstruction_error.cpu().data.item()) error_rel3d.append(rel_error.cpu().data.item()) error_relf.append(f_error.cpu().data.item()) print(f"f/sequence: {f_test}/{j} | f/fgt: {f[0].item():.3f}/{fgt.item():.3f} | f_error_rel: {f_error.item():.4f} | rmse: {reconstruction_error.item():.4f} | rel rmse: {rel_error.item():.4f} | 2d error: {reproj_error.item():.4f}") avg_2d = np.mean(error_2d) avg_rel3d = np.mean(error_rel3d) avg_3d = np.mean(error_3d) avg_relf = np.mean(error_relf) seterror_2d.append(avg_2d) seterror_3d.append(avg_3d) seterror_rel3d.append(avg_rel3d) seterror_relf.append(avg_relf) out_f.append(np.stack(f_pred)) out_shape.append(np.stack(shape_pred,axis=0)) print(f"f_error_rel: {avg_relf:.4f} | rel rmse: {avg_rel3d:.4f} | 2d error: {reproj_error.item():.4f} | rmse: {avg_3d:.4f} |") out_shape = np.stack(out_shape) out_f = np.stack(out_f) all_f = np.stack(all_f).flatten() all_fpred = np.stack(all_fpred).flatten() all_d = np.stack(all_depth).flatten() allerror_2d = np.stack(allerror_2d).flatten() allerror_3d = np.stack(allerror_3d).flatten() allerror_rel3d = np.stack(allerror_rel3d).flatten() matdata = {} matdata['fvals'] = np.array(f_vals) matdata['all_f'] = np.array(all_f) matdata['all_fpred'] = np.array(all_fpred) matdata['all_d'] = np.array(all_depth) matdata['error_2d'] = allerror_2d matdata['error_3d'] = allerror_3d matdata['error_rel3d'] = allerror_rel3d matdata['seterror_2d'] = np.array(seterror_2d) matdata['seterror_3d'] = np.array(seterror_3d) matdata['seterror_rel3d'] = np.array(seterror_rel3d) matdata['seterror_relf'] = np.array(seterror_relf) matdata['shape'] = np.stack(out_shape) matdata['f'] = np.stack(out_f) scipy.io.savemat(outfile,matdata) print(f"MEAN seterror_2d: {np.mean(seterror_2d)}") print(f"MEAN seterror_3d: {np.mean(seterror_3d)}") print(f"MEAN seterror_rel3d: {np.mean(seterror_rel3d)}") print(f"MEAN seterror_relf: {np.mean(seterror_relf)}")
def dualoptimization(ptsI, calib_net, sfm_net, shape_gt=None, fgt=None, M=100, N=68, mode='still', ptstart=0): if mode == 'still': alpha = 0.1 else: alpha = 0.001 # define what weights gets optimized calib_net.eval() sfm_net.eval() trainfc(calib_net) trainfc(sfm_net) M, _, N = ptsI.shape # run the model f = calib_net(ptsI) + 300 f = f.mean() betas = sfm_net(ptsI) betas = betas.unsqueeze(-1) eigenvec = torch.stack(M * [lm_eigenvec]) shape = torch.stack(M * [mu_lm]) + torch.bmm( eigenvec, betas).squeeze().view(M, N, 3) shape = shape - shape.mean(1).unsqueeze(1) shape = shape.mean(0) opt1 = torch.optim.Adam(calib_net.parameters(), lr=1e-5) opt2 = torch.optim.Adam(sfm_net.parameters(), lr=1) curloss = 100 for outerloop in itertools.count(): shape = shape.detach() for iter in itertools.count(): opt1.zero_grad() f = calib_net(ptsI) + 300 f = f.mean() K = torch.zeros(3, 3).float() K[0, 0] = f K[1, 1] = f K[2, 2] = 1 # pose estimation km, c_w, scaled_betas, alphas = util.EPnP(ptsI, shape, K) _, R, T, mask = util.optimizeGN(km, c_w, scaled_betas, alphas, shape, ptsI, K) Xc = torch.bmm(R, torch.stack(M * [shape.T])) + T.unsqueeze(2) #shape_error = util.getShapeError(ptsI,Xc,shape,f,R,T) error_time = util.getTimeConsistency(shape, R, T) error2d = util.getReprojError2(ptsI, shape, R, T, K, show=False, loss='l2') # apply loss loss = error2d.mean() #loss = error2d.mean() + alpha*error_time if iter >= 5: break prv_loss = loss.item() loss.backward() opt1.step() # log results on console if not shape_gt is None: rmse = torch.norm(shape_gt - shape, dim=1).mean().item() else: rmse = -1 if not fgt is None: ftrue = fgt.item() else: fgt = -1 f_error = torch.mean(torch.abs(f - ftrue)) print( f"iter: {iter} | error: {loss.item():.3f} | f/fgt: {f.item():.1f}/{ftrue:.1f} | error2d: {error2d.mean().item():.3f} | error_time: {error_time.item():.2f} | rmse: {rmse:.2f}" ) f = f.detach() for iter in itertools.count(): opt2.zero_grad() # shape prediction betas = sfm_net(ptsI) betas = betas.unsqueeze(-1) eigenvec = torch.stack(M * [lm_eigenvec]) shape = torch.stack(M * [mu_lm]) + torch.bmm( eigenvec, betas).squeeze().view(M, N, 3) shape = shape - shape.mean(1).unsqueeze(1) shape = shape.mean(0) K = torch.zeros((3, 3)).float() K[0, 0] = f K[1, 1] = f K[2, 2] = 1 # differentiable PnP pose estimation km, c_w, scaled_betas, alphas = util.EPnP(ptsI, shape, K) _, R, T, mask = util.optimizeGN(km, c_w, scaled_betas, alphas, shape, ptsI, K) error2d = util.getReprojError2(ptsI, shape, R, T, K, show=False, loss='l2') Xc = torch.bmm(R, torch.stack(M * [shape.T])) + T.unsqueeze(2) #shape_error = util.getShapeError(ptsI,Xc,shape,f,R,T) error_time = util.getTimeConsistency(shape, R, T) # apply loss loss = error2d.mean() #loss = error2d.mean() + alpha*error_time #if iter >= 5 and loss > prv_loss: break if iter >= 5: break loss.backward() opt2.step() prv_loss = loss.item() # log results on console if not shape_gt is None: rmse = torch.norm(shape_gt - shape, dim=1).mean().item() else: rmse = -1 if not fgt is None: ftrue = fgt.item() else: fgt = -1 f_error = torch.mean(torch.abs(f - ftrue)) print( f"iter: {iter} | error: {loss.item():.3f} | f/fgt: {f.item():.1f}/{ftrue:.1f} | error2d: {error2d.mean().item():.3f} | error_time: {error_time.item():.2f} | rmse: {rmse:.2f}" ) if torch.abs(curloss - loss) <= 0.01 or curloss < loss: break curloss = loss return shape, K, R, T
def dualoptimization(ptsI, calib_net, sfm_net, shape_gt=None, fgt=None, M=100, N=68, mode='still', ptstart=0, db='real'): if mode == 'still': alpha = 1 else: alpha = 0.001 # define what weights gets optimized calib_net.eval() sfm_net.eval() #calib_net.train() #sfm_net.train() trainfc(calib_net) trainfc(sfm_net) M = ptsI.shape[0] xmin, _ = torch.min(ptsI[:, 0, :], dim=1) xmax, _ = torch.max(ptsI[:, 0, :], dim=1) ymin, _ = torch.min(ptsI[:, 1, :], dim=1) ymax, _ = torch.max(ptsI[:, 1, :], dim=1) width = torch.abs(xmin - xmax) height = torch.abs(ymin - ymax) area = width * height # run the model #ptsI = x.squeeze().permute(1,0).reshape((M,N,2)).permute(0,2,1) f = torch.squeeze(calib_net(ptsI) + 300) betas = sfm_net(ptsI) betas = betas.unsqueeze(-1) eigenvec = torch.stack(M * [lm_eigenvec]) shape = torch.stack(M * [mu_lm]) + torch.bmm( eigenvec, betas).squeeze().view(M, N, 3) shape = shape - shape.mean(1).unsqueeze(1) shape = shape.mean(0) opt1 = torch.optim.Adam(calib_net.parameters(), lr=1e-4) opt2 = torch.optim.Adam(sfm_net.parameters(), lr=1e-1) curloss = 100000 for outerloop in itertools.count(): shape = shape.detach() for iter in itertools.count(): opt1.zero_grad() f = torch.squeeze(calib_net(ptsI) + 300) #f = f.mean() K = torch.zeros(M, 3, 3).float() K[:, 0, 0] = f K[:, 1, 1] = f K[:, 2, 2] = 1 # pose estimation km, c_w, scaled_betas, alphas = util.EPnP_single(ptsI, shape, K) _, R, T, mask = util.optimizeGN(km, c_w, scaled_betas, alphas, shape, ptsI) Xc = torch.bmm(R, torch.stack(M * [shape.T])) + T.unsqueeze(2) # get time error error_time = util.getTimeConsistency(shape, R, T) # get shape consistency error_s = util.getShapeError(ptsI, Xc, shape, f, R, T) # get 2D reprojection error error2d = util.getError(ptsI, shape, R, T, K, show=False, loss='l2') # get relative depth error error3d = util.getDepthError(area, shape, R, T) # apply loss #loss = error2d.mean() #loss = error2d.mean() + error3d loss = error2d.mean() + error_time * alpha #loss = error_s + error2d.mean() #loss = error_s #loss = error_s + error_time*alpha #loss = error3d if iter >= 5: break prv_loss = loss.item() loss.backward() opt1.step() # log results on console if not shape_gt is None: rmse = torch.norm(shape_gt - shape, dim=1).mean().item() else: rmse = -1 if not fgt is None: ftrue = fgt.item() else: fgt = -1 f_error = torch.mean(torch.abs(f - ftrue)) print( f"iter: {iter} | error: {loss.item():.3f} | f/fgt: {f.mean().item():.3f}/{f.median().item():.3f}/{f.std().item():.3f}/{fgt.item():.3f} | f_error: {f_error.item():.1f} | error2d: {error2d.mean().item():.3f} | rmse: {rmse:.2f}" ) f = f.detach() for iter in itertools.count(): opt2.zero_grad() # shape prediction betas = sfm_net(ptsI) betas = betas.unsqueeze(-1) eigenvec = torch.stack(M * [lm_eigenvec]) shape = torch.stack(M * [mu_lm]) + torch.bmm( eigenvec, betas).squeeze().view(M, N, 3) shape = shape - shape.mean(1).unsqueeze(1) shape = shape.mean(0) K = torch.zeros(M, 3, 3).float() K[:, 0, 0] = f K[:, 1, 1] = f K[:, 2, 2] = 1 # differentiable PnP pose estimation km, c_w, scaled_betas, alphas = util.EPnP_single(ptsI, shape, K) _, R, T, mask = util.optimizeGN(km, c_w, scaled_betas, alphas, shape, ptsI) Xc = torch.bmm(R, torch.stack(M * [shape.T])) + T.unsqueeze(2) # get time error error_time = util.getTimeConsistency(shape, R, T) # get shape consistency error_s = util.getShapeError(ptsI, Xc, shape, f, R, T) # error2d error2d = util.getError(ptsI, shape, R, T, K, show=False, loss='l2') #Xc = torch.bmm(R,torch.stack(M*[shape.T])) + T.unsqueeze(2) # get relative depth error error3d = util.getDepthError(area, shape, R, T) # apply loss #loss = error2d.mean() #loss = error2d.mean()+ error3d loss = error2d.mean() + error_time * alpha #loss = error_s + error2d.mean() #loss = error_s #loss = error_s + error_time*alpha #loss = error3d if iter >= 5: break loss.backward() opt2.step() prv_loss = loss.item() # log results on console if not shape_gt is None: rmse = torch.norm(shape_gt - shape, dim=1).mean().item() else: rmse = -1 if not fgt is None: ftrue = fgt.item() else: fgt = -1 f_error = torch.mean(torch.abs(f - ftrue)) print( f"iter: {iter} | error: {loss.item():.3f} | f/fgt: {f.mean().item():.3f}/{f.median().item():.3f}/{f.std().item():.3f}/{fgt.item():.3f} | f_error: {f_error.item():.1f} | error2d: {error2d.mean().item():.3f} | rmse: {rmse:.2f}" ) #print(f"iter: {iter} | error: {loss.item():.3f} | f/fgt: {f.mean().item():.3f}/{fgt.item():.3f} | f_error: {f_error.item():.1f} | error2d: {error2d.mean().item():.3f} | rmse: {rmse:.2f}") #print(f"iter: {iter} | error: {loss.item():.3f} | f_error: {f_error:.1f} | error2d: {error2d.mean().item():.3f} | rmse: {rmse:.2f}") if torch.abs(curloss - loss) <= 0.01 or curloss < loss: break curloss = loss return shape, K, R, T, outerloop
def test(modelin=args.model, outfile=args.out, optimize=args.opt): # define model, dataloader, 3dmm eigenvectors, optimization method calib_net = PointNet(n=1) sfm_net = PointNet(n=199) if modelin != "": calib_path = os.path.join('model', 'calib_' + modelin) sfm_path = os.path.join('model', 'sfm_' + modelin) calib_net.load_state_dict(torch.load(calib_path)) sfm_net.load_state_dict(torch.load(sfm_path)) calib_net.eval() sfm_net.eval() 68 // 10 Mvals = [i for i in range(1, 100)] Nvals = [i for i in range(3, 68)] f_vals = [i * 200 for i in range(2, 7)] fpred_mean = np.zeros((100, 65, 5, 5)) fpred_med = np.zeros((100, 65, 5, 5)) fpred = np.zeros((100, 65, 5, 5)) factual = np.zeros((100, 65, 5, 5)) depth_error = np.zeros((100, 65, 5, 5)) for i, viewcount in enumerate(Mvals): for j, ptcount in enumerate(Nvals): for l, ftest in enumerate(f_vals): data3dmm = dataloader.MNLoader(M=viewcount, N=ptcount, f=ftest, seed=0) M = data3dmm.M N = data3dmm.N # mean shape and eigenvectors for 3dmm mu_s = torch.from_numpy(data3dmm.mu_s).float().detach() mu_s[:, 2] = mu_s[:, 2] * -1 lm_eigenvec = torch.from_numpy( data3dmm.lm_eigenvec).float().detach() sigma = torch.from_numpy(data3dmm.sigma).float().detach() sigma = torch.diag(sigma.squeeze()) lm_eigenvec = torch.mm(lm_eigenvec, sigma) mu_s = torch.from_numpy(data3dmm.mu_s).float().detach() mu_s[:, 2] = mu_s[:, 2] * -1 lm_eigenvec = torch.from_numpy( data3dmm.lm_eigenvec).float().detach() sigma = torch.from_numpy(data3dmm.sigma).float().detach() sigma = torch.diag(sigma.squeeze()) lm_eigenvec = torch.mm(lm_eigenvec, sigma) for k in range(5): data = data3dmm[k] # load the data x_cam_gt = data['x_cam_gt'] shape_gt = data['x_w_gt'] fgt = data['f_gt'] x_img = data['x_img'] x_img_gt = data['x_img_gt'] ptsI = x_img.reshape((M, N, 2)).permute(0, 2, 1) x = x_img.unsqueeze(0).permute(0, 2, 1) # run the model f = torch.squeeze(calib_net(ptsI) + 300) betas = sfm_net(ptsI) betas = betas.unsqueeze(-1) eigenvec = torch.stack(M * [lm_eigenvec]) shape = torch.stack(M * [mu_s]) + torch.bmm( eigenvec, betas).squeeze().view(M, N, 3) shape = shape - shape.mean(1).unsqueeze(1) shape = shape.mean(0) # get motion measurement guess K = torch.zeros((M, 3, 3)).float() K[:, 0, 0] = f K[:, 1, 1] = f K[:, 2, 2] = 1 km, c_w, scaled_betas, alphas = util.EPnP_single( ptsI, shape, K) _, R, T, mask = util.optimizeGN(km, c_w, scaled_betas, alphas, shape, ptsI) error_time = util.getTimeConsistency(shape, R, T) if error_time > 20: mode = 'walk' else: mode = 'still' # apply dual optimization if optimize: calib_net.load_state_dict(torch.load(calib_path)) sfm_net.load_state_dict(torch.load(sfm_path)) shape, K, R, T = dualoptimization(x, calib_net, sfm_net, lm_eigenvec, betas, mu_s, shape_gt=shape_gt, fgt=fgt, M=M, N=N, mode=mode) f = K[0, 0].detach() # get motion measurement guess fmu = f.mean() fmed = f.flatten().median() K = torch.zeros(M, 3, 3).float() K[:, 0, 0] = fmu K[:, 1, 1] = fmu K[:, 2, 2] = 1 K, _ = torch.median(K, dim=0) km, c_w, scaled_betas, alphas = util.EPnP(ptsI, shape, K) Xc, R, T, mask = util.optimizeGN(km, c_w, scaled_betas, alphas, shape, ptsI) # get errors rel_errors = util.getRelReprojError3(x_cam_gt, shape, R, T) fpred_mean[i, j, l, k] = fmu.detach().cpu().item() fpred_med[i, j, l, k] = fmed.detach().cpu().item() factual[i, j, l, k] = fgt.detach().cpu().item() depth_error[i, j, l, k] = rel_errors.cpu().mean().item() print( f"M: {viewcount} | N: {ptcount} | f/fgt: {fpred_mean[i,j,l,k]:.2f}/{factual[i,j,l,k]}" ) ferror_mu = np.mean( np.abs(fpred_mean[i, j, l] - factual[i, j, l]) / factual[i, j, l]) ferror_med = np.mean( np.abs(fpred_med[i, j, l] - factual[i, j, l]) / factual[i, j, l]) derror = np.mean(depth_error[i, j, l]) f = np.mean(fpred_mean[i, j, l]) print( f"M: {viewcount} | N: {ptcount} | fgt: {ftest:.2f} | ferror_mu: {ferror_mu:.2f} | ferror_med: {ferror_med:.2f} | derror: {derror:.2f}" ) matdata = {} matdata['fpred_mu'] = fpred_mean matdata['fpred_med'] = fpred_med matdata['fgt'] = factual matdata['derror'] = depth_error scipy.io.savemat(outfile, matdata) print(f"saved output to {outfile}")
def testReal(modelin=args.model, outfile=args.out, optimize=args.opt, db=args.db): # define model, dataloader, 3dmm eigenvectors, optimization method calib_net = PointNet(n=1) sfm_net = PointNet(n=199) if modelin != "": calib_path = os.path.join('model', 'calib_' + modelin) sfm_path = os.path.join('model', 'sfm_' + modelin) calib_net.load_state_dict(torch.load(calib_path)) sfm_net.load_state_dict(torch.load(sfm_path)) calib_net.eval() sfm_net.eval() # mean shape and eigenvectors for 3dmm data3dmm = dataloader.SyntheticLoader() mu_lm = torch.from_numpy(data3dmm.mu_lm).float().detach() mu_lm[:, 2] = mu_lm[:, 2] * -1 lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float().detach() sigma = torch.from_numpy(data3dmm.sigma).float().detach() sigma = torch.diag(sigma.squeeze()) lm_eigenvec = torch.mm(lm_eigenvec, sigma) # define loader loader = getLoader(db) out_fpred = [] out_fgt = [] out_dpred = [] out_dgt = [] shape_pred = [] error_2d = [] error_relf = [] error_rel3d = [] for sub in range(len(loader)): batch = loader[sub] x_cam_gt = batch['x_cam_gt'] fgt = batch['f_gt'] x_img = batch['x_img'] x_img_gt = batch['x_img_gt'] M = x_img_gt.shape[0] N = x_img_gt.shape[-1] ptsI = x_img.reshape((M, N, 2)).permute(0, 2, 1) x = x_img.unsqueeze(0).permute(0, 2, 1) # run the model f = torch.squeeze(calib_net(ptsI) + 300) betas = sfm_net(ptsI) betas = betas.unsqueeze(-1) eigenvec = torch.stack(M * [lm_eigenvec]) shape = torch.stack(M * [mu_lm]) + torch.bmm( eigenvec, betas).squeeze().view(M, N, 3) shape = shape - shape.mean(1).unsqueeze(1) shape = shape.mean(0) # get motion measurement guess K = torch.zeros((M, 3, 3)).float() K[:, 0, 0] = f K[:, 1, 1] = f K[:, 2, 2] = 1 km, c_w, scaled_betas, alphas = util.EPnP_single(ptsI, shape, K) _, R, T, mask = util.optimizeGN(km, c_w, scaled_betas, alphas, shape, ptsI) error_time = util.getTimeConsistency(shape, R, T) if error_time > 20: mode = 'walk' else: mode = 'still' print(mode, error_time) # additional optimization on initial solution shape_gt = batch['x_w_gt'] if db == 'biwi' else None if optimize: calib_net.load_state_dict(torch.load(calib_path)) sfm_net.load_state_dict(torch.load(sfm_path)) print(mode) if db == 'biwi': shape, K, R, T, iter = dualoptimization(ptsI, calib_net, sfm_net, shape_gt=shape_gt, fgt=fgt, db='biwi', mode=mode) else: shape, K, R, T, iter = dualoptimization(ptsI, calib_net, sfm_net, fgt=fgt, mode=mode) f = K[:, 0, 0].detach() # get pose with single intrinsic fmu = f.mean() fmed = f.flatten().median() K = torch.zeros(M, 3, 3).float() K[:, 0, 0] = fmu K[:, 1, 1] = fmu K[:, 2, 2] = 1 km, c_w, scaled_betas, alphas = util.EPnP_single(ptsI, shape, K) Xc, R, T, mask = util.optimizeGN(km, c_w, scaled_betas, alphas, shape, ptsI) # get errors #reproj_errors2 = util.getError(ptsI,shape,R,T,K) reproj_errors2 = util.getReprojError2(ptsI, shape, R, T, K.mean(0), show=False, loss='l2') rel_errors = util.getRelReprojError3(x_cam_gt, shape, R, T) d = torch.norm(T, dim=1) dgt = torch.norm(torch.mean(x_cam_gt, dim=2), dim=1) reproj_error = reproj_errors2.mean() rel_error = rel_errors.mean() f_error = torch.mean(torch.abs(fgt - fmu) / fgt) # save final prediction out_fpred.append(f.detach().cpu().numpy()) out_fgt.append(fgt.numpy()) out_dpred.append(d.detach().cpu().numpy()) out_dgt.append(dgt.cpu().numpy()) f_x = torch.mean(fmu.detach()).cpu().item() shape_pred.append(shape.detach().cpu().numpy()) error_2d.append(reproj_error.cpu().data.item()) error_rel3d.append(rel_error.cpu().data.item()) error_relf.append(f_error.cpu().data.item()) print( f" f/fgt: {f_x:.3f}/{fgt.item():.3f} | f_error_rel: {f_error.item():.4f} | rel rmse: {rel_error.item():.4f} | 2d error: {reproj_error.item():.4f}" ) #end for # prepare output file out_shape = np.stack(shape_pred) out_fpred = np.array(out_fpred, dtype=np.object) out_fgt = np.array(out_fgt, dtype=np.object).T matdata = {} matdata['fpred'] = out_fpred matdata['fgt'] = out_fgt matdata['dpred'] = out_dpred matdata['dgt'] = out_dgt matdata['shape'] = np.stack(out_shape) matdata['error_2d'] = np.array(error_2d) matdata['error_rel3d'] = np.array(error_rel3d) matdata['error_relf'] = np.array(error_relf) scipy.io.savemat(outfile, matdata) print(f"MEAN seterror_2d: {np.mean(error_2d)}") print(f"MEAN seterror_rel3d: {np.mean(error_rel3d)}") print(f"MEAN seterror_relf: {np.mean(error_relf)}")
def test(modelin=args.model,outfile=args.out,optimize=args.opt): # define model, dataloader, 3dmm eigenvectors, optimization method calib_net = CalibrationNet3(n=1) sfm_net = CalibrationNet3(n=199) if modelin != "": calib_path = os.path.join('model','calib_' + modelin) sfm_path = os.path.join('model','sfm_' + modelin) calib_net.load_state_dict(torch.load(calib_path,map_location='cpu')) sfm_net.load_state_dict(torch.load(sfm_path,map_location='cpu')) calib_net.to(args.device) sfm_net.to(args.device) calib_net.eval() sfm_net.eval() # mean shape and eigenvectors for 3dmm M = 100 data3dmm = dataloader.SyntheticLoader() mu_lm = torch.from_numpy(data3dmm.mu_lm).float().to(args.device).detach() mu_lm[:,2] = mu_lm[:,2]*-1 lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float().to(args.device).detach() sigma = torch.from_numpy(data3dmm.sigma).float().to(args.device).detach() sigma = torch.diag(sigma.squeeze()) lm_eigenvec = torch.mm(lm_eigenvec, sigma) batch_size = 10 lm_eigenvec = torch.stack(batch_size*[lm_eigenvec]) # sample from f testing set allerror_2d = [] allerror_3d = [] allerror_rel3d = [] allerror_relf = [] all_f = [] all_fpred = [] all_depth = [] out_shape = [] out_f = [] seterror_3d = [] seterror_rel3d = [] seterror_relf = [] seterror_2d = [] f_vals = [i*100 for i in range(4,15)] for f_test in f_vals: f_test = 1000 # create dataloader data = dataloader.TestData() data.batchsize = batch_size loader = data.createLoader(f_test) # containers f_pred = [] shape_pred = [] error_2d = [] error_3d = [] error_rel3d = [] error_relf = [] M = 100; N = 68; batch_size = data.batchsize; for j,data in enumerate(loader): # load the data x_cam_gt = data['x_cam_gt'].to(args.device) shape_gt = data['x_w_gt'].to(args.device) fgt = data['f_gt'].to(args.device) x_img = data['x_img'].to(args.device) x_img_gt = data['x_img_gt'].to(args.device) T_gt = data['T_gt'].to(args.device) # reshape and form data one = torch.ones(batch_size,M*N,1).to(device=args.device) x_img_one = torch.cat([x_img,one],dim=2) x_cam_pt = x_cam_gt.permute(0,1,3,2).reshape(batch_size,6800,3) x = x_img.permute(0,2,1).reshape(batch_size,2,M,N) ptsI = x_img_one.reshape(batch_size,M,N,3).permute(0,1,3,2)[:,:,:2,:] # run the model f = calib_net(x) + 300 betas = sfm_net(x) betas = betas.squeeze(0).unsqueeze(-1) shape = mu_lm + torch.bmm(lm_eigenvec,betas).squeeze().view(batch_size,N,3) # additional optimization on initial solution if optimize: calib_net.load_state_dict(torch.load(calib_path,map_location=args.device)) sfm_net.load_state_dict(torch.load(sfm_path,map_location=args.device)) calib_net.train() sfm_net.train() opt1 = torch.optim.Adam(calib_net.parameters(),lr=1e-4) opt2 = torch.optim.Adam(sfm_net.parameters(),lr=1e-2) curloss = 100 for outerloop in itertools.count(): # camera calibration shape = shape.detach() for iter in itertools.count(): opt1.zero_grad() f = torch.mean(calib_net.forward2(x) + 300) K = torch.zeros(3,3).float().to(device=args.device) K[0,0] = f K[1,1] = f K[2,2] = 1 # ground truth l1 error f_error = torch.mean(torch.abs(f - fgt)) # rmse rmse = torch.norm(shape_gt - shape,dim=2).mean() # differentiable PnP pose estimation error1 = [] for i in range(batch_size): km, c_w, scaled_betas, alphas = util.EPnP(ptsI[i],shape[i],K) Xc, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape[i],ptsI[i],K) error2d = util.getReprojError2(ptsI[i],shape[i],R,T,K,show=False,loss='l1') error1.append(error2d.mean()) # loss loss = torch.stack(error1).mean() # stopping condition if iter == 5: break if iter > 5 and prev_loss < loss: break else: prev_loss = loss # update loss.backward() opt1.step() print(f"iter: {iter} | error: {loss.item():.3f} | f/fgt: {f.mean().item():.1f}/{fgt.mean().item():.1f} | error2d: {loss.item():.3f} | rmse: {rmse.item():.3f} ") # sfm f = f.detach() for iter in itertools.count(): opt2.zero_grad() # shape prediction betas = sfm_net.forward2(x) betas = betas.unsqueeze(-1) shape = mu_lm + torch.bmm(lm_eigenvec,betas).squeeze().view(batch_size,N,3) K = torch.zeros((3,3)).float() K[0,0] = f K[1,1] = f K[2,2] = 1 #rmse = torch.norm(shape_gt - shape,dim=1).mean().detach() rmse = torch.norm(shape_gt - shape,dim=2).mean() # differentiable PnP pose estimation error1 = [] for i in range(batch_size): km, c_w, scaled_betas, alphas = util.EPnP(ptsI[i],shape[i],K) Xc, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape[i],ptsI[i],K) error2d = util.getReprojError2(ptsI[i],shape[i],R,T,K,show=False,loss='l1') errorTime = util.getTimeConsistency(shape[i],R,T) error1.append(error2d.mean()) #loss = torch.stack(error1).mean() + 0.01*torch.stack(error2).mean() loss = torch.stack(error1).mean() if iter == 5: break if iter > 5 and prev_loss < loss: break else: prev_loss = loss loss.backward() opt2.step() print(f"iter: {iter} | error: {loss.item():.3f} | f/fgt: {f.mean().item():.1f}/{fgt.mean().item():.1f} | error2d: {loss.item():.3f} | rmse: {rmse.item():.3f} ") # closing condition for outerloop on dual objective if torch.abs(curloss - loss) < 0.01: break curloss = loss else: K = torch.zeros((batch_size,3,3)).float().to(device=args.device) K[:,0,0] = f.squeeze() K[:,1,1] = f.squeeze() K[:,2,2] = 1 km,c_w,scaled_betas,alphas = util.EPnP(ptsI,shape,K) Xc, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI,K) #all_fpred.append(batch_size*[f.detach().item()]) e2d,e3d,eshape,e2d_all,e3d_all,d_all = util.getBatchError(ptsI.detach(),shape.detach(),K.detach(),x_cam_gt,shape_gt) f_error = torch.squeeze(torch.abs(fgt - f)/fgt) e2d = e2d.cpu().numpy() e3d = e3d.cpu().numpy() eshape = eshape.cpu().numpy() f_error = f_error.cpu().squeeze().numpy() e2d_all = e2d_all.cpu().numpy() e3d_all = e3d_all.cpu().numpy() d_all = d_all.cpu().numpy() f_pred.append(f.detach().cpu().item()) shape_pred.append(shape.detach().cpu().numpy()) all_depth.append(d_all.flatten()) all_f.append(np.array([fgt.mean()] * d_all.flatten().shape[0])) all_fpred.append(np.array([f.mean()]*d_all.flatten().shape[0])) print(f"f/sequence: {f_test}/{j} | f/fgt: {f.mean().item():.3f}/{fgt.mean().item():.3f} | f_error_rel: {f_error.mean().item():.4f} | rmse: {eshape.mean().item():.4f} | rel rmse: {np.mean(e3d):.4f} | 2d error: {np.mean(e2d):.4f}") avg_2d = np.mean(error_2d) avg_rel3d = np.mean(error_rel3d) avg_3d = np.mean(error_3d) avg_relf = np.mean(error_relf) seterror_2d.append(avg_2d) seterror_3d.append(avg_3d) seterror_rel3d.append(avg_rel3d) seterror_relf.append(avg_relf) out_f.append(np.array(f_pred)) out_shape.append(np.concatenate(shape_pred,axis=0)) print(f"f_error_rel: {avg_relf:.4f} | rel rmse: {avg_rel3d:.4f} | 2d error: {avg_2d:.4f} | rmse: {avg_3d:.4f} |") out_shape = np.stack(out_shape) out_f = np.stack(out_f) all_f = np.stack(all_f).flatten() all_fpred = np.stack(all_fpred).flatten() all_depth = np.stack(all_depth).flatten() allerror_2d = np.stack(allerror_2d).flatten() allerror_rel3d = np.stack(allerror_rel3d).flatten() matdata = {} matdata['fvals'] = np.array(f_vals) matdata['all_f'] = np.array(all_f) matdata['all_fpred'] = np.array(all_fpred) matdata['all_d'] = np.array(all_depth) matdata['error_2d'] = allerror_2d matdata['error_rel3d'] = allerror_rel3d matdata['seterror_2d'] = np.array(seterror_2d) matdata['seterror_3d'] = np.array(seterror_3d) matdata['seterror_rel3d'] = np.array(seterror_rel3d) matdata['seterror_relf'] = np.array(seterror_relf) matdata['out_shape'] = out_shape matdata['out_f'] = out_f scipy.io.savemat(outfile,matdata) print(f"MEAN seterror_2d: {np.mean(seterror_2d)}") print(f"MEAN seterror_3d: {np.mean(seterror_3d)}") print(f"MEAN seterror_rel3d: {np.mean(seterror_rel3d)}") print(f"MEAN seterror_relf: {np.mean(seterror_relf)}")