def testReal(modelin=args.model,outfile=args.out,optimize=args.opt,db=args.db):
# mean shape and eigenvectors for 3dmm
data3dmm = dataloader.SyntheticLoader()
mu_lm = torch.from_numpy(data3dmm.mu_lm).float().detach()
mu_lm[:,2] = mu_lm[:,2]*-1
lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float().detach()
sigma = torch.from_numpy(data3dmm.sigma).float().detach()
sigma = torch.diag(sigma.squeeze())
lm_eigenvec = torch.mm(lm_eigenvec, sigma)
# create bpnp sfm model
sfm_net = torchvision.models.vgg11()
sfm_net.classifier = torch.nn.Linear(25088,68*3)
# define loader and initialize logging variables
loader = getLoader(db)
f_pred = []
d_pred = []
shape_pred = []
error_2d = []
error_relf = []
error_rel3d = []
for idx in range(len(loader)):
# load the data
data = loader[idx]
x_cam_gt = data['x_cam_gt']
fgt = data['f_gt']
x_img = data['x_img']
x_img_gt = data['x_img_gt']
M = x_img_gt.shape[0]
N = 68
# create bpnp camera calibration/sfm model
calib_net= (1.1*torch.randn(1)).requires_grad_()
sfm_net.load_state_dict(torch.load('model/bpnp_sfm.net'))
ptsI = x_img.reshape((M,N,2)).permute(0,2,1)
x2d = x_img.view((M,N,2))
x_img_pts = x_img.reshape((M,N,2)).permute(0,2,1)
one = torch.ones(M*N,1)
x_img_one = torch.cat([x_img,one],dim=1)
x = x_img_one.permute(1,0)
# run the model
f = torch.sigmoid(calib_net)*2000
shape = mu_lm
ini_pose = torch.zeros((M,6))
ini_pose[:,5] = 99
curloss = 100
# apply dual optimization
shape,K,R,T = dualoptimization(x,ptsI,x2d,ini_pose,calib_net,sfm_net,fgt=fgt)
f = K[0,0].detach()
# get final result to save
depth = torch.norm(x_cam_gt.mean(2),dim=1)
# get errors
reproj_errors2 = util.getReprojError2(ptsI,shape,R,T,K)
rel_errors = util.getRelReprojError3(x_cam_gt,shape,R,T)
reproj_error = reproj_errors2.mean()
rel_error = rel_errors.mean()
f_error = torch.abs(fgt - f) / fgt
# save final prediction
f_pred.append(f.detach().cpu().item())
d_pred.append(depth.detach().cpu().numpy())
shape_pred.append(shape.detach().cpu().numpy())
error_2d.append(reproj_error.cpu().data.item())
error_rel3d.append(rel_error.cpu().data.item())
error_relf.append(f_error.cpu().data.item())
print(f" f/fgt: {f.item():.3f}/{fgt.item():.3f} | f_error_rel: {f_error.item():.4f} | rel rmse: {rel_error.item():.4f} | 2d error: {reproj_error.item():.4f}")
# prepare output file
matdata = {}
matdata['f_pred'] = np.stack(f_pred)
matdata['d_pred'] = np.concatenate(d_pred)
matdata['shape_pred'] = np.stack(shape_pred)
matdata['error_2d'] = np.array(error_2d)
matdata['error_relf'] = np.array(error_relf)
matdata['error_rel3d'] = np.stack(error_rel3d)
scipy.io.savemat(outfile,matdata)
print(f"MEAN seterror_2d: {np.mean(error_2d)}")
print(f"MEAN seterror_rel3d: {np.mean(error_rel3d)}")
print(f"MEAN seterror_relf: {np.mean(error_relf)}")
def test(modelin=args.model,outfile=args.out,feature_transform=args.ft):
# define model, dataloader, 3dmm eigenvectors, optimization method
#if modelin != "":
# model.load_state_dict(torch.load(modelin))
#model.eval()
#model.cuda()
# mean shape and eigenvectors for 3dmm
M = 100
N = 68
data3dmm = dataloader.SyntheticLoader()
mu_lm = torch.from_numpy(data3dmm.mu_lm).float()
mu_lm[:,2] = mu_lm[:,2]*-1
le = torch.mean(mu_lm[36:42,:],axis=0)
re = torch.mean(mu_lm[42:48,:],axis=0)
ipd = torch.norm(le - re)
lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float()
#optimizer = torch.optim.Adam(model.parameters(),lr=1e-2)
# sample from f testing set
allerror_2d = []
allerror_3d = []
allerror_rel3d = []
allerror_relf = []
all_f = []
all_fpred = []
all_depth = []
out_shape = []
out_f = []
seterror_3d = []
seterror_rel3d = []
seterror_relf = []
seterror_2d = []
f_vals = [i*100 for i in range(4,15)]
# set random seed for reproducibility of test set
for f_test in f_vals:
# create dataloader
loader = dataloader.TestLoader(f_test)
f_pred = []
shape_pred = []
error_2d = []
error_3d = []
error_rel3d = []
error_relf = []
M = 100;
N = 68;
batch_size = 1;
for j, data in enumerate(loader):
if j == 10: break
# create bpnp camera calibration model
calib_net= (1.1*torch.randn(1)).requires_grad_()
# create bpnp sfm model
sfm_net = torchvision.models.vgg11()
sfm_net.classifier = torch.nn.Linear(25088,N*3)
# load the data
x_cam_gt = data['x_cam_gt']
shape_gt = data['x_w_gt']
fgt = data['f_gt']
x_img = data['x_img']
x_img_gt = data['x_img_gt']
depth = torch.norm(x_cam_gt.mean(2),dim=1)
all_depth.append(depth.numpy())
all_f.append(fgt.numpy()[0])
ptsI = x_img.reshape((M,N,2)).permute(0,2,1)
x_img_pts = x_img.reshape((M,N,2)).permute(0,2,1)
one = torch.ones(M*N,1)
x_img_one = torch.cat([x_img,one],dim=1)
x = x_img_one.permute(1,0)
# run the model
f = torch.sigmoid(calib_net)*2000
shape = mu_lm
ini_pose = torch.zeros((M,6))
ini_pose[:,5] = 99
curloss = 100
# apply dual optimization
shape,K,R,T = dualoptimization(x,ptsI,x2d,ini_pose,calib_net,sfm_net,shape_gt=shape_gt,fgt=fgt)
f = K[0,0].detach()
all_fpred.append(f.item())
# get errors
km,c_w,scaled_betas, alphas = util.EPnP(ptsI,shape,K)
Xc, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI,K)
# get errors
reproj_errors2 = util.getReprojError2(ptsI,shape,R,T,K)
reproj_errors3 = torch.norm(shape_gt - shape,dim=1).mean()
rel_errors = util.getRelReprojError3(x_cam_gt,shape,R,T)
reproj_error = reproj_errors2.mean()
reconstruction_error = reproj_errors3.mean()
rel_error = rel_errors.mean()
f_error = torch.abs(fgt - f) / fgt
# save final prediction
f_pred.append(f.detach().cpu().item())
shape_pred.append(shape.detach().cpu().numpy())
allerror_3d.append(reproj_error.data.numpy())
allerror_2d.append(reconstruction_error.data.numpy())
allerror_rel3d.append(rel_error.data.numpy())
error_2d.append(reproj_error.cpu().data.item())
error_3d.append(reconstruction_error.cpu().data.item())
error_rel3d.append(rel_error.cpu().data.item())
error_relf.append(f_error.cpu().data.item())
print(f"f/sequence: {f_test}/{j} | f/fgt: {f.item():.3f}/{fgt.item():.3f} | f_error_rel: {f_error.item():.4f} | rmse: {reconstruction_error.item():.4f} | rel rmse: {rel_error.item():.4f} | 2d error: {reproj_error.item():.4f}")
#end for
avg_2d = np.mean(error_2d)
avg_rel3d = np.mean(error_rel3d)
avg_3d = np.mean(error_3d)
avg_relf = np.mean(error_relf)
seterror_2d.append(avg_2d)
seterror_3d.append(avg_3d)
seterror_rel3d.append(avg_rel3d)
seterror_relf.append(avg_relf)
out_f.append(np.stack(f_pred))
out_shape.append(np.stack(shape_pred,axis=0))
#end for
all_f = np.stack(all_f).flatten()
all_fpred = np.stack(all_fpred).flatten()
all_d = np.stack(all_depth).flatten()
allerror_2d = np.stack(allerror_2d).flatten()
allerror_3d = np.stack(allerror_3d).flatten()
allerror_rel3d = np.stack(allerror_rel3d).flatten()
matdata = {}
#matdata['shape'] = shape.detach().cpu().numpy()
matdata['fvals'] = np.array(f_vals)
matdata['all_f'] = np.array(all_f)
matdata['all_fpred'] = np.array(all_fpred)
matdata['all_d'] = np.array(all_depth)
matdata['error_2d'] = allerror_2d
matdata['error_3d'] = allerror_3d
matdata['error_rel3d'] = allerror_rel3d
matdata['seterror_2d'] = np.array(seterror_2d)
matdata['seterror_3d'] = np.array(seterror_3d)
matdata['seterror_rel3d'] = np.array(seterror_rel3d)
matdata['seterror_relf'] = np.array(seterror_relf)
matdata['shape'] = np.stack(out_shape)
matdata['f'] = np.stack(out_f)
scipy.io.savemat(outfile,matdata)
print(f"MEAN seterror_2d: {np.mean(seterror_2d)}")
print(f"MEAN seterror_3d: {np.mean(seterror_3d)}")
print(f"MEAN seterror_rel3d: {np.mean(seterror_rel3d)}")
print(f"MEAN seterror_relf: {np.mean(seterror_relf)}")
def testBIWI(model,modelin=args.model,outfile=args.out,feature_transform=args.feat_trans):
if modelin != "":
model.load_state_dict(torch.load(modelin))
model.eval()
# load 3dmm data
data3dmm = dataloader.SyntheticLoader()
mu_lm = torch.from_numpy(data3dmm.mu_lm).float()
lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float()
shape = mu_lm
shape[:,2] = shape[:,2] * -1
loader = dataloader.BIWILoader()
seterror_3d = []
seterror_rel3d = []
seterror_relf = []
seterror_2d = []
for sub in range(len(loader)):
batch = loader[sub]
x_cam_gt = batch['x_cam_gt']
x_w_gt = batch['x_w_gt']
f_gt = batch['f_gt']
x_img = batch['x_img']
x_img_gt = batch['x_img_gt']
M = x_img_gt.shape[0]
one = torch.ones(M,1,68)
x_img_one = torch.cat([x_img,one],dim=1)
# run the model
out, trans, transfeat = model(x_img_one)
alphas = out[:,:199].mean(0)
f = torch.relu(out[:,199]).mean()
K = torch.zeros((3,3)).float()
K[0,0] = f;
K[1,1] = f;
K[2,2] = 1;
K[0,2] = 320;
K[1,2] = 240;
Xc,R,T = util.EPnP(x_img,shape,K)
# apply 3DMM model from predicted parameters
reproj_errors2 = util.getReprojError2(x_img,shape,R,T,K)
reproj_errors3 = util.getReprojError3(x_cam_gt,shape,R,T)
rel_errors = util.getRelReprojError3(x_cam_gt,shape,R,T)
reproj_error = reproj_errors2.mean()
reconstruction_error = reproj_errors3.mean()
rel_error = rel_errors.mean()
f_error = torch.abs(f_gt - f) / f_gt
seterror_2d.append(reproj_error.cpu().data.item())
seterror_3d.append(reconstruction_error.cpu().data.item())
seterror_rel3d.append(rel_error.cpu().data.item())
seterror_relf.append(f_error.cpu().data.item())
print(f"fgt: {f_gt.mean().item():.3f} | f_error_rel: {f_error.item():.4f} | rmse: {reconstruction_error.item():.4f} | rel rmse: {rel_error.item():.4f} | 2d error: {reproj_error.item():.4f}")
#end for
matdata = {}
matdata['seterror_2d'] = np.array(seterror_2d)
matdata['seterror_3d'] = np.array(seterror_3d)
matdata['seterror_rel3d'] = np.array(seterror_rel3d)
matdata['seterror_relf'] = np.array(seterror_relf)
scipy.io.savemat(outfile,matdata)
print(f"MEAN seterror_2d: {np.mean(seterror_2d)}")
print(f"MEAN seterror_3d: {np.mean(seterror_3d)}")
print(f"MEAN seterror_rel3d: {np.mean(seterror_rel3d)}")
print(f"MEAN seterror_relf: {np.mean(seterror_relf)}")
def test(model, modelin=args.model,outfile=args.out,feature_transform=args.feat_trans):
# define model, dataloader, 3dmm eigenvectors, optimization method
if modelin != "":
model.load_state_dict(torch.load(modelin))
model.cuda()
# mean shape and eigenvectors for 3dmm
M = 100
data3dmm = dataloader.SyntheticLoader()
mu_lm = torch.from_numpy(data3dmm.mu_lm).float().cuda()
lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float().cuda()
# sample from f testing set
allerror_2d = []
allerror_3d = []
allerror_rel3d = []
allerror_relf = []
allerror_f = []
allerror_d = []
seterror_3d = []
seterror_rel3d = []
seterror_relf = []
seterror_2d = []
f_vals = [i*100 for i in range(4,21)]
for f_test in f_vals:
# create dataloader
data = dataloader.TestLoader(f_test)
error_2d = []
error_3d = []
error_rel3d = []
error_relf = []
for k in range(len(data)):
batch = data[k]
x_cam_gt = batch['x_cam_gt'].cuda()
x_w_gt = batch['x_w_gt'].cuda()
f_gt = batch['f_gt'].cuda()
x_img = batch['x_img'].cuda()
x_img_gt = batch['x_img_gt'].cuda()
T_gt = batch['T_gt']
allerror_d.append(T_gt[:,2])
one = torch.ones(M,1,68).cuda()
x_img_one = torch.cat([x_img,one],dim=1)
# run the model
out, trans, transfeat = model(x_img_one)
alphas = out[:,:199].mean(0)
f = torch.relu(out[:,199]).mean()
K = torch.zeros((3,3)).float().cuda()
for f = np.linspace(-200,200,100):
K[0,0] = f;
K[1,1] = f;
K[2,2] = 1;
K[0,2] = 320;
K[1,2] = 240;
# apply 3DMM model from predicted parameters
alpha_matrix = torch.diag(alphas)
shape_cov = torch.mm(lm_eigenvec,alpha_matrix)
s = shape_cov.sum(1).view(68,3)
#shape = (mu_lm + s)
shape = mu_lm
shape[:,2] = shape[:,2]*-1
# run epnp algorithm
# get control points
c_w = util.getControlPoints(shape)
# solve alphas
alphas = util.solveAlphas(shape,c_w)
# setup M
px = 320;
py = 240;
Matrix = util.setupM(alphas,x_img.permute(0,2,1),px,py,f)
# get eigenvectors of M for each view
u,d,v = torch.svd(Matrix)
#solve N=1
c_c_n1 = v[:,:,-1].reshape((100,4,3)).permute(0,2,1)
_ , x_c_n1, _ = util.scaleControlPoints(c_c_n1,c_w[:3,:],alphas,shape)
Rn1,Tn1 = util.getExtrinsics(x_c_n1,shape)
reproj_error2_n1 = util.getReprojError2(x_img,shape,Rn1,Tn1,K)
reproj_error3_n1 = util.getReprojError3(x_cam_gt,shape,Rn1,Tn1)
rel_error_n1 = util.getRelReprojError3(x_cam_gt,shape,Rn1,Tn1)
# solve N=2
# get distance contraints
d12,d13,d14,d23,d24,d34 = util.getDistances(c_w)
distances = torch.stack([d12,d13,d14,d23,d24,d34])**2
beta_n2 = util.getBetaN2(v[:,:,-2:],distances)
c_c_n2 = util.getControlPointsN2(v[:,:,-2:],beta_n2)
_,x_c_n2,_ = util.scaleControlPoints(c_c_n2,c_w[:3,:],alphas,shape)
Rn2,Tn2 = util.getExtrinsics(x_c_n2,shape)
reproj_error2_n2 = util.getReprojError2(x_img,shape,Rn2,Tn2,K)
reproj_error3_n2 = util.getReprojError3(x_cam_gt,shape,Rn2,Tn2)
rel_error_n2 = util.getRelReprojError3(x_cam_gt,shape,Rn1,Tn1)
mask = reproj_error2_n1 < reproj_error2_n2
reproj_errors = torch.cat((reproj_error2_n1[mask],reproj_error2_n2[~mask]))
rmse_errors = torch.cat((reproj_error3_n1[mask],reproj_error3_n2[~mask]))
rel_errors = torch.cat((rel_error_n2[~mask],rel_error_n1[mask]))
print(rel_errors.mean())
quit()
# errors
allerror_3d.append(reproj_errors.cpu().data.numpy())
allerror_2d.append(rmse_errors.cpu().data.numpy())
allerror_rel3d.append(rel_errors.cpu().data.numpy())
reproj_error = torch.mean(reproj_errors)
reconstruction_error = torch.mean(rmse_errors)
rel_error = torch.mean(rel_errors)
f_error = torch.abs(f_gt - f) / f_gt
error_2d.append(reproj_error.cpu().data.item())
error_3d.append(reconstruction_error.cpu().data.item())
error_rel3d.append(rel_error.cpu().data.item())
error_relf.append(f_error.cpu().data.item())
print(f"f/sequence: {f_test}/{k} | f_error_rel: {f_error.item():.4f} | rmse: {reconstruction_error.item():.4f} | rel rmse: {rel_error.item():.4f} | 2d error: {reproj_error.item():.4f}")
#end for
avg_2d = np.mean(error_2d)
avg_rel3d = np.mean(error_rel3d)
avg_3d = np.mean(error_3d)
avg_relf = np.mean(error_relf)
seterror_2d.append(avg_2d)
seterror_3d.append(avg_3d)
seterror_rel3d.append(avg_rel3d)
seterror_relf.append(avg_relf)
def train(modelin=args.model, modelout=args.out,device=args.device,opt=args.opt,ft=args.ft):
# define model, dataloader, 3dmm eigenvectors, optimization method
calib_net = PointNet(n=1,feature_transform=ft)
sfm_net = PointNet(n=199,feature_transform=ft)
if modelin != "":
calib_path = os.path.join('model','calib_' + modelin)
sfm_path = os.path.join('model','sfm_' + modelin)
pretrained1 = torch.load(calib_path)
pretrained2 = torch.load(sfm_path)
calib_dict = calib_net.state_dict()
sfm_dict = sfm_net.state_dict()
pretrained1 = {k: v for k,v in pretrained1.items() if k in calib_dict}
pretrained2 = {k: v for k,v in pretrained2.items() if k in sfm_dict}
calib_dict.update(pretrained1)
sfm_dict.update(pretrained2)
calib_net.load_state_dict(pretrained1)
sfm_net.load_state_dict(pretrained2)
calib_net.to(device=device)
sfm_net.to(device=device)
opt1 = torch.optim.Adam(calib_net.parameters(),lr=1e-3)
opt2 = torch.optim.Adam(sfm_net.parameters(),lr=1e-3)
# dataloader
loader = dataloader.SyntheticLoader()
batch_size = 100
M = loader.M
N = loader.N
# mean shape and eigenvectors for 3dmm
mu_lm = torch.from_numpy(loader.mu_lm).float()#.to(device=device)
mu_lm[:,2] = mu_lm[:,2] * -1
mu_lm = torch.stack(300 * [mu_lm.to(device=device)])
shape = mu_lm
lm_eigenvec = torch.from_numpy(loader.lm_eigenvec).float().to(device=device)
sigma = torch.from_numpy(loader.sigma).float().detach().to(device=device)
sigma = torch.diag(sigma.squeeze())
lm_eigenvec = torch.mm(lm_eigenvec, sigma)
lm_eigenvec = torch.stack(300 * [lm_eigenvec])
# main training loop
best = 10000
for epoch in itertools.count():
for i in range(len(loader)):
if i < 3: continue
v1 = loader[i]
v2 = loader[i-1]
v3 = loader[i-2]
batch = stackVideos([v1,v2,v3],100,68,device=device)
# get the input and gt values
alpha_gt = batch['alpha']
x_cam_gt = batch['x_cam_gt']
shape_gt = batch['x_w_gt']
fgt = batch['f_gt']
x = batch['x_img']
M = x.shape[0]
N = x.shape[-1]
# calibration
f = torch.squeeze(calib_net(x) + 300)
K = torch.zeros((M,3,3)).float().to(device=device)
K[:,0,0] = f
K[:,1,1] = f
K[:,2,2] = 1
# sfm
alpha = sfm_net(x)
alpha = alpha.unsqueeze(-1)
shape = mu_lm + torch.bmm(lm_eigenvec,alpha).squeeze().view(M,N,3)
shape[0:100] = shape[0:100] - shape[0:100].mean(1).unsqueeze(1)
shape[100:200] = shape[100:200] - shape[100:200].mean(1).unsqueeze(1)
shape[200:300] = shape[200:300] - shape[200:300].mean(1).unsqueeze(1)
opt1.zero_grad()
opt2.zero_grad()
f1_error = torch.mean(torch.abs(f[0:100] - fgt[0]))
f2_error = torch.mean(torch.abs(f[100:200] - fgt[1]))
f3_error = torch.mean(torch.abs(f[200:300] - fgt[2]))
#a1_error = torch.mean(torch.abs(alpha[0:100] - alpha_gt[0]))
#a2_error = torch.mean(torch.abs(alpha[100:200] - alpha_gt[1]))
#a3_error = torch.mean(torch.abs(alpha[200:300] - alpha_gt[2]))
s1_error = torch.mean(torch.abs(shape[0:100] - shape_gt[0].unsqueeze(0)))
s2_error = torch.mean(torch.abs(shape[100:200] - shape_gt[1].unsqueeze(0)))
s3_error = torch.mean(torch.abs(shape[200:300] - shape_gt[2].unsqueeze(0)))
ferror = f1_error + f2_error + f3_error
#aerror = a1_error + a2_error + a3_error
serror = s1_error + s2_error + s3_error
#f_error = torch.mean(torch.abs(f - fgt))
#error3d = torch.mean(torch.norm(shape - shape_gt,dim=2))
#error = ferror + aerror
error = ferror + serror
error.backward()
opt1.step()
opt2.step()
print(f"iter: {i} | best: {best:.2f} | f_error: {ferror.item():.3f} | serror: {serror.item():.3f} ")
if i == 1000: break
# save model and increment weight decay
torch.save(sfm_net.state_dict(), os.path.join('model','sfm_model.pt'))
torch.save(calib_net.state_dict(), os.path.join('model','calib_model.pt'))
ferror = test(modelin='model.pt',outfile=args.out,optimize=False)
if ferror < best:
best = ferror
print("saving!")
torch.save(sfm_net.state_dict(), os.path.join('model','sfm_'+modelout))
torch.save(calib_net.state_dict(), os.path.join('model','calib_'+modelout))
sfm_net.train()
calib_net.train()
def testReal(modelin=args.model,outfile=args.out,optimize=args.opt,db=args.db):
# define model, dataloader, 3dmm eigenvectors, optimization method
calib_net = PointNet(n=1)
sfm_net = PointNet(n=199)
if modelin != "":
calib_path = os.path.join('model','calib_' + modelin)
sfm_path = os.path.join('model','sfm_' + modelin)
calib_net.load_state_dict(torch.load(calib_path))
sfm_net.load_state_dict(torch.load(sfm_path))
calib_net.eval()
sfm_net.eval()
# mean shape and eigenvectors for 3dmm
data3dmm = dataloader.SyntheticLoader()
mu_lm = torch.from_numpy(data3dmm.mu_lm).float().detach()
mu_lm[:,2] = mu_lm[:,2]*-1
lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float().detach()
sigma = torch.from_numpy(data3dmm.sigma).float().detach()
sigma = torch.diag(sigma.squeeze())
lm_eigenvec = torch.mm(lm_eigenvec, sigma)
# define loader
loader = getLoader(db)
f_pred = []
shape_pred = []
error_2d = []
error_relf = []
error_rel3d = []
for sub in range(len(loader)):
batch = loader[sub]
x_cam_gt = batch['x_cam_gt']
fgt = batch['f_gt']
x_img = batch['x_img']
x_img_gt = batch['x_img_gt']
M = x_img_gt.shape[0]
N = x_img_gt.shape[-1]
ptsI = x_img.reshape((M,N,2)).permute(0,2,1)
x = x_img.unsqueeze(0).permute(0,2,1)
# run the model
f = calib_net(x) + 300
betas = sfm_net(x)
betas = betas.squeeze(0).unsqueeze(-1)
shape = mu_lm + torch.mm(lm_eigenvec,betas).squeeze().view(N,3)
shape = shape - shape.mean(0).unsqueeze(0)
# get motion measurement guess
K = torch.zeros((3,3)).float()
K[0,0] = f
K[1,1] = f
K[2,2] = 1
km,c_w,scaled_betas,alphas = util.EPnP(ptsI,shape,K)
_, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI)
error_time = util.getTimeConsistency(shape,R,T)
if error_time > 20:
mode='walk'
else:
mode='still'
# adjust number of landmarks
M = x_img_gt.shape[0]
N = x_img_gt.shape[-1]
# additional optimization on initial solution
if optimize:
calib_net.load_state_dict(torch.load(calib_path))
sfm_net.load_state_dict(torch.load(sfm_path))
if db == 'biwi':
shape_gt = batch['x_w_gt']
shape,K,R,T = dualoptimization(x,calib_net,sfm_net,shape_gt=shape_gt,fgt=fgt,M=M,N=N,mode=mode,db='biwi')
else:
shape,K,R,T = dualoptimization(x,calib_net,sfm_net,fgt=fgt,M=M,N=N,mode=mode)
f = K[0,0].detach()
else:
K = torch.zeros(3,3).float()
K[0,0] = f
K[1,1] = f
K[2,2] = 1
km,c_w,scaled_betas,alphas = util.EPnP(ptsI,shape,K)
Xc, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI)
# get errors
reproj_errors2 = util.getReprojError2(ptsI,shape,R,T,K)
rel_errors = util.getRelReprojError3(x_cam_gt,shape,R,T)
reproj_error = reproj_errors2.mean()
rel_error = rel_errors.mean()
f_error = torch.abs(fgt - f) / fgt
# save final prediction
f_pred.append(f.detach().cpu().item())
shape_pred.append(shape.detach().cpu().numpy())
error_2d.append(reproj_error.cpu().data.item())
error_rel3d.append(rel_error.cpu().data.item())
error_relf.append(f_error.cpu().data.item())
print(f" f/fgt: {f.item():.3f}/{fgt.item():.3f} | f_error_rel: {f_error.item():.4f} | rel rmse: {rel_error.item():.4f} | 2d error: {reproj_error.item():.4f}")
#end for
# prepare output file
out_shape = np.stack(shape_pred)
out_f = np.stack(f_pred)
matdata = {}
matdata['shape'] = np.stack(out_shape)
matdata['f'] = np.stack(out_f)
matdata['error_2d'] = np.array(error_2d)
matdata['error_rel3d'] = np.array(error_rel3d)
matdata['error_relf'] = np.array(error_relf)
scipy.io.savemat(outfile,matdata)
print(f"MEAN seterror_2d: {np.mean(error_2d)}")
print(f"MEAN seterror_rel3d: {np.mean(error_rel3d)}")
print(f"MEAN seterror_relf: {np.mean(error_relf)}")
def train(modelin=args.model,
modelout=args.out,
device=args.device,
opt=args.opt):
# define model, dataloader, 3dmm eigenvectors, optimization method
calib_net = PointNet(n=1, N=68)
sfm_net = PointNet(n=199, N=68)
if modelin != "":
calib_path = os.path.join('model', 'calib_' + modelin)
sfm_path = os.path.join('model', 'sfm_' + modelin)
pretrained1 = torch.load(calib_path)
pretrained2 = torch.load(sfm_path)
calib_dict = calib_net.state_dict()
sfm_dict = sfm_net.state_dict()
pretrained1 = {k: v for k, v in pretrained1.items() if k in calib_dict}
pretrained2 = {k: v for k, v in pretrained2.items() if k in sfm_dict}
calib_dict.update(pretrained1)
sfm_dict.update(pretrained2)
calib_net.load_state_dict(pretrained1)
sfm_net.load_state_dict(pretrained2)
calib_net.to(device=device)
sfm_net.to(device=device)
opt1 = torch.optim.Adam(calib_net.parameters(), lr=1e-4)
opt2 = torch.optim.Adam(sfm_net.parameters(), lr=1e-2)
# dataloader
loader = dataloader.SyntheticLoader()
#data = dataloader.Data()
#loader = data.batchloader
#batch_size = data.batchsize
batch_size = 100
# mean shape and eigenvectors for 3dmm
#data3dmm = dataloader.SyntheticLoader()
#mu_lm = torch.from_numpy(data3dmm.mu_lm).float().detach()
#mu_lm[:,2] = mu_lm[:,2]*-1
#lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float().detach()
#sigma = torch.from_numpy(data3dmm.sigma).float().detach()
#sigma = torch.diag(sigma.squeeze())
#lm_eigenvec = torch.mm(lm_eigenvec, sigma)
mu_lm = torch.from_numpy(loader.mu_lm).float()
mu_lm[:, 2] = mu_lm[:, 2] * -1
mu_lm = torch.stack(batch_size * [mu_lm])
shape = mu_lm
lm_eigenvec = torch.from_numpy(
loader.lm_eigenvec).float().to(device=device)
lm_eigenvec = torch.stack(batch_size * [lm_eigenvec])
M = 100
N = 68
# main training loop
best = 1000
for epoch in itertools.count():
for j, batch in enumerate(loader):
# get the input and gt values
x_cam_gt = batch['x_cam_gt'].to(device=device)
shape_gt = batch['x_w_gt'].to(device=device)
fgt = batch['f_gt'].to(device=device)
x_img = batch['x_img'].to(device=device)
#beta_gt = batch['beta_gt'].to(device=device)
#x_img_norm = batch['x_img_norm']
x_img_gt = batch['x_img_gt'].to(device=device).permute(1, 0, 2)
batch_size = fgt.shape[0]
# train single view model
#x = x_img.permute(1,0)
#x = x_img.permute(0,2,1).reshape(batch_size,2,M,N)
ptsI = x_img.reshape(M, N, 2).permute(0, 2, 1)
x = ptsI
# if just optimizing
if not opt:
# calibration
f = calib_net(x) + 300
f = f.mean()
K = torch.zeros((M, 3, 3)).float().to(device=device)
K[:, 0, 0] = f.squeeze()
K[:, 1, 1] = f.squeeze()
K[:, 2, 2] = 1
# sfm
betas = sfm_net(x)
betas = betas.unsqueeze(-1)
shape = mu_lm + torch.bmm(lm_eigenvec, betas).squeeze().view(
M, N, 3)
shape = shape - shape.mean(1).unsqueeze(1)
shape = shape.mean(0)
opt1.zero_grad()
opt2.zero_grad()
f_error = torch.mean(torch.abs(f - fgt))
#error2d = torch.mean(torch.abs(pred - x_img_gt))
error3d = torch.mean(torch.abs(shape - shape_gt))
error = f_error + error3d
error.backward()
opt1.step()
opt2.step()
print(
f"f_error: {f_error.item():.3f} | error3d: {error3d.item():.3f} | f/fgt: {f.item():.1f}/{fgt.item():.1f} "
)
continue
# save model and increment weight decay
torch.save(sfm_net.state_dict(), os.path.join('model', 'sfm_model.pt'))
torch.save(calib_net.state_dict(),
os.path.join('model', 'calib_model.pt'))
ferror = test(modelin='model.pt', outfile=args.out, optimize=False)
if ferror < best:
best = ferror
print("saving!")
torch.save(sfm_net.state_dict(),
os.path.join('model', 'sfm_' + modelout))
torch.save(calib_net.state_dict(),
os.path.join('model', 'calib_' + modelout))
sfm_net.train()
calib_net.train()
def train(modelin=args.model, modelout=args.out):
# define logger
#torch.manual_seed(6)
#if log:
# logger = Logger(logname)
# define model, dataloader, 3dmm eigenvectors, optimization method
torch.manual_seed(2)
calib_net = Model1(k=1, feature_transform=False)
sfm_net = Model1(k=199, feature_transform=False)
#if modelin != "":
# model.load_state_dict(torch.load(modelin))
opt1 = torch.optim.Adam(calib_net.parameters(), lr=1e-1)
opt2 = torch.optim.Adam(sfm_net.parameters(), lr=1e-1)
# dataloader
#data = dataloader.Data()
#loader = data.batchloader
#loader = dataloader.BIWILoader()
loader = dataloader.SyntheticLoader()
# mean shape and eigenvectors for 3dmm
mu_lm = torch.from_numpy(loader.mu_lm).float()
#mu_lm[:,2] = mu_lm[:,2] * -1
shape = mu_lm
lm_eigenvec = torch.from_numpy(loader.lm_eigenvec).float()
# main training loop
for epoch in itertools.count():
for j, data in enumerate(loader):
M = loader.M
N = loader.N
# get the input and gt values
x_cam_gt = data['x_cam_gt']
shape_gt = data['x_w_gt']
fgt = data['f_gt']
x_img = data['x_img']
x_img_gt = data['x_img_gt']
x_img_pts = x_img.reshape((M, N, 2)).permute(0, 2, 1)
one = torch.ones(M * N, 1)
x_img_one = torch.cat([x_img, one], dim=1)
x_cam_pt = x_cam_gt.permute(0, 2, 1).reshape(M * N, 3)
x = x_img_one.permute(1, 0)
# get initial values for betas and alphas of EPNP
ptsI = x_img.reshape((M, N, 2)).permute(0, 2, 1)
fvals = []
errors = []
for outerloop in itertools.count():
# calibration
shape = shape.detach()
for iter in itertools.count():
opt1.zero_grad()
# focal length prediction
f, _, _ = calib_net(x.unsqueeze(0))
f = f + 300
K = torch.zeros((3, 3)).float()
K[0, 0] = f
K[1, 1] = f
K[2, 2] = 1
# RMSE between GT and predicted shape
rmse = torch.norm(shape_gt - shape, dim=1).mean().detach()
# error f
error_f = torch.mean(torch.abs(f - fgt))
# differentiable PnP pose estimation
km, c_w, scaled_betas, alphas = util.EPnP(ptsI, shape, K)
Xc, R, T, mask = util.optimizeGN(km, c_w, scaled_betas,
alphas, shape, ptsI, K)
error2d = util.getReprojError2(ptsI,
shape,
R,
T,
K,
show=False,
loss='l1')
loss = error2d.mean() + error_f
if iter > 20 and prev_loss < loss:
break
else:
prev_loss = loss
loss.backward()
opt1.step()
print(
f"iter: {iter} | error: {loss.item():.3f} | f/fgt: {f.item():.1f}/{fgt[0].item():.1f} | rmse: {rmse.item():.2f}"
)
# sfm
f = f.detach()
for iter in itertools.count():
opt2.zero_grad()
# shape prediction
betas, _, _ = sfm_net(x.unsqueeze(0))
shape = torch.sum(betas * lm_eigenvec, 1)
shape = shape.reshape(68, 3) + mu_lm
K = torch.zeros((3, 3)).float()
K[0, 0] = f
K[1, 1] = f
K[2, 2] = 1
# RMSE between GT and predicted shape
rmse = torch.norm(shape_gt - shape, dim=1).mean().detach()
# differentiable PnP pose estimation
km, c_w, scaled_betas, alphas = util.EPnP(ptsI, shape, K)
Xc, R, T, mask = util.optimizeGN(km, c_w, scaled_betas,
alphas, shape, ptsI, K)
error2d = util.getReprojError2(ptsI,
shape,
R,
T,
K,
show=False,
loss='l2')
loss = error2d.mean()
if iter > 20 and prev_loss < loss:
break
else:
prev_loss = loss
loss.backward()
opt2.step()
print(
f"iter: {iter} | error: {loss.item():.3f} | f/fgt: {f.item():.1f}/{fgt[0].item():.1f} | rmse: {rmse.item():.2f}"
)
if outerloop == 2: break
# get errors
reproj_errors2 = util.getReprojError2(ptsI, shape, R, T, K)
reproj_errors3 = util.getReprojError3(x_cam_gt, shape, R, T)
rel_errors = util.getRelReprojError3(x_cam_gt, shape, R, T)
reproj_error = reproj_errors2.mean()
reconstruction_error = reproj_errors3.mean()
rel_error = rel_errors.mean()
f_error = torch.abs(fgt - f) / fgt
print(
f"f/fgt: {f[0].item():.3f}/{fgt.item():.3f} | f_error_rel: {f_error.item():.4f} | rmse: {reconstruction_error.item():.4f} | rel rmse: {rel_error.item():.4f} | 2d error: {reproj_error.item():.4f}"
)
#end for
torch.save(sfm_net.state_dict(),
os.path.join('model', 'sfm_' + modelout))
torch.save(calib_net.state_dict(),
os.path.join('model', 'calib_' + modelout))
def testReal(modelin=args.model,
outfile=args.out,
optimize=args.opt,
db=args.db):
# define model, dataloader, 3dmm eigenvectors, optimization method
calib_net = PointNet(n=1)
sfm_net = PointNet(n=199)
if modelin != "":
calib_path = os.path.join('model', 'calib_' + modelin)
sfm_path = os.path.join('model', 'sfm_' + modelin)
calib_net.load_state_dict(torch.load(calib_path))
sfm_net.load_state_dict(torch.load(sfm_path))
calib_net.eval()
sfm_net.eval()
# mean shape and eigenvectors for 3dmm
data3dmm = dataloader.SyntheticLoader()
mu_lm = torch.from_numpy(data3dmm.mu_lm).float().detach()
mu_lm[:, 2] = mu_lm[:, 2] * -1
lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float().detach()
sigma = torch.from_numpy(data3dmm.sigma).float().detach()
sigma = torch.diag(sigma.squeeze())
lm_eigenvec = torch.mm(lm_eigenvec, sigma)
# define loader
loader = getLoader(db)
out_fpred = []
out_fgt = []
out_dpred = []
out_dgt = []
shape_pred = []
error_2d = []
error_relf = []
error_rel3d = []
for sub in range(len(loader)):
batch = loader[sub]
x_cam_gt = batch['x_cam_gt']
fgt = batch['f_gt']
x_img = batch['x_img']
x_img_gt = batch['x_img_gt']
M = x_img_gt.shape[0]
N = x_img_gt.shape[-1]
ptsI = x_img.reshape((M, N, 2)).permute(0, 2, 1)
x = x_img.unsqueeze(0).permute(0, 2, 1)
# run the model
f = torch.squeeze(calib_net(ptsI) + 300)
betas = sfm_net(ptsI)
betas = betas.unsqueeze(-1)
eigenvec = torch.stack(M * [lm_eigenvec])
shape = torch.stack(M * [mu_lm]) + torch.bmm(
eigenvec, betas).squeeze().view(M, N, 3)
shape = shape - shape.mean(1).unsqueeze(1)
shape = shape.mean(0)
# get motion measurement guess
K = torch.zeros((M, 3, 3)).float()
K[:, 0, 0] = f
K[:, 1, 1] = f
K[:, 2, 2] = 1
km, c_w, scaled_betas, alphas = util.EPnP_single(ptsI, shape, K)
_, R, T, mask = util.optimizeGN(km, c_w, scaled_betas, alphas, shape,
ptsI)
error_time = util.getTimeConsistency(shape, R, T)
if error_time > 20:
mode = 'walk'
else:
mode = 'still'
print(mode, error_time)
# additional optimization on initial solution
shape_gt = batch['x_w_gt'] if db == 'biwi' else None
if optimize:
calib_net.load_state_dict(torch.load(calib_path))
sfm_net.load_state_dict(torch.load(sfm_path))
print(mode)
if db == 'biwi':
shape, K, R, T, iter = dualoptimization(ptsI,
calib_net,
sfm_net,
shape_gt=shape_gt,
fgt=fgt,
db='biwi',
mode=mode)
else:
shape, K, R, T, iter = dualoptimization(ptsI,
calib_net,
sfm_net,
fgt=fgt,
mode=mode)
f = K[:, 0, 0].detach()
# get pose with single intrinsic
fmu = f.mean()
fmed = f.flatten().median()
K = torch.zeros(M, 3, 3).float()
K[:, 0, 0] = fmu
K[:, 1, 1] = fmu
K[:, 2, 2] = 1
km, c_w, scaled_betas, alphas = util.EPnP_single(ptsI, shape, K)
Xc, R, T, mask = util.optimizeGN(km, c_w, scaled_betas, alphas, shape,
ptsI)
# get errors
#reproj_errors2 = util.getError(ptsI,shape,R,T,K)
reproj_errors2 = util.getReprojError2(ptsI,
shape,
R,
T,
K.mean(0),
show=False,
loss='l2')
rel_errors = util.getRelReprojError3(x_cam_gt, shape, R, T)
d = torch.norm(T, dim=1)
dgt = torch.norm(torch.mean(x_cam_gt, dim=2), dim=1)
reproj_error = reproj_errors2.mean()
rel_error = rel_errors.mean()
f_error = torch.mean(torch.abs(fgt - fmu) / fgt)
# save final prediction
out_fpred.append(f.detach().cpu().numpy())
out_fgt.append(fgt.numpy())
out_dpred.append(d.detach().cpu().numpy())
out_dgt.append(dgt.cpu().numpy())
f_x = torch.mean(fmu.detach()).cpu().item()
shape_pred.append(shape.detach().cpu().numpy())
error_2d.append(reproj_error.cpu().data.item())
error_rel3d.append(rel_error.cpu().data.item())
error_relf.append(f_error.cpu().data.item())
print(
f" f/fgt: {f_x:.3f}/{fgt.item():.3f} | f_error_rel: {f_error.item():.4f} | rel rmse: {rel_error.item():.4f} | 2d error: {reproj_error.item():.4f}"
)
#end for
# prepare output file
out_shape = np.stack(shape_pred)
out_fpred = np.array(out_fpred, dtype=np.object)
out_fgt = np.array(out_fgt, dtype=np.object).T
matdata = {}
matdata['fpred'] = out_fpred
matdata['fgt'] = out_fgt
matdata['dpred'] = out_dpred
matdata['dgt'] = out_dgt
matdata['shape'] = np.stack(out_shape)
matdata['error_2d'] = np.array(error_2d)
matdata['error_rel3d'] = np.array(error_rel3d)
matdata['error_relf'] = np.array(error_relf)
scipy.io.savemat(outfile, matdata)
print(f"MEAN seterror_2d: {np.mean(error_2d)}")
print(f"MEAN seterror_rel3d: {np.mean(error_rel3d)}")
print(f"MEAN seterror_relf: {np.mean(error_relf)}")
def train(modelin=args.model,
modelout=args.out,
log=args.log,
logname=args.logname):
# define logger
if log:
logger = Logger(logname)
# define model, dataloader, 3dmm eigenvectors, optimization method
#torch.manual_seed(2)
model = Model1(k=1, feature_transform=True)
if modelin != "":
model.load_state_dict(torch.load(modelin))
model #.cuda()
optimizer = torch.optim.Adam(model.parameters(), lr=1e-1)
# dataloader
#data = dataloader.Data()
#loader = data.batchloader
loader = dataloader.SyntheticLoader()
# mean shape and eigenvectors for 3dmm
mu_lm = torch.from_numpy(loader.mu_lm).float() #.cuda()
mu_lm[:, 2] = mu_lm[:, 2] * -1
shape = mu_lm.detach() #.cuda()
lm_eigenvec = torch.from_numpy(loader.lm_eigenvec).float() #.cuda()
M = loader.M
N = loader.N
# main training loop
for epoch in itertools.count():
#for j,batch in enumerate(loader):
#np.random.seed(0)
for j, data in enumerate(loader):
# get the input and gt values
x_cam_gt = data['x_cam_gt'] #.cuda()
x_w_gt = data['x_w_gt'] #.cuda()
fgt = data['f_gt'] #.cuda()
x_img = data['x_img'] #.cuda()
x_img_gt = data['x_img_gt'] #.cuda()
x_img_pts = x_img.reshape((M, N, 2)).permute(0, 2, 1)
one = torch.ones(M * N, 1) #.cuda()
x_img_one = torch.cat([x_img, one], dim=1)
x_cam_pt = x_cam_gt.permute(0, 2, 1).reshape(M * N, 3)
# run the model
x = x_img_one.permute(1, 0)
# get initial values for betas and alphas of EPNP
ptsI = x_img.reshape((M, N, 2)).permute(0, 2, 1)
#km, c_w, scaled_betas, alphas = util.EPnP(ptsI,shape,K)
#Xc,R,T, scaled_betas = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI,K)
#loss = util.getReprojError2(ptsI,shape,R,T,K).mean()
# optimize using EPNP+GN
fvals = []
errors = []
for iter in itertools.count():
optimizer.zero_grad()
# model output
f, _, _ = model(x.unsqueeze(0))
f = torch.nn.functional.leaky_relu(f) + 300
K = torch.zeros((3, 3)).float()
K[0, 0] = f
K[1, 1] = f
K[2, 2] = 1
# differentiable pose estimation
km, c_w, scaled_betas, alphas = util.EPnP(ptsI, shape, K)
Xc, R, T, _ = util.optimizeGN(km, c_w, scaled_betas, alphas,
shape, ptsI, K)
error2d = util.getReprojError2(ptsI,
shape,
R,
T,
K,
show=False).mean()
errorT = util.getTConsistency(T)
errorR = util.getRConsistency(R)
#error3dconsistency = util.get3DConsistency(ptsI,shape,kinv,R,T)
#loss = error2d + errorT*0.001 + errorR
loss = error2d
loss.backward()
optimizer.step()
errors.append(loss.detach().cpu().item())
fvals.append(f.detach().cpu().item())
data = {}
data['ptsI'] = ptsI.detach().cpu().numpy()
data['shape'] = shape.detach().cpu().numpy()
data['R'] = R.detach().cpu().numpy()
data['T'] = T.detach().cpu().numpy()
data['Xc'] = Xc.detach().cpu().numpy()
data['K'] = K.detach().cpu().numpy()
data['fvals'] = np.array(fvals)
data['loss'] = np.array(errors)
scipy.io.savemat(f"visual/shape{iter:03d}.mat", data)
print(
f"iter: {iter} | error: {loss.item():.3f} | f/fgt: {f.item():.1f}/{fgt[0].item():.1f}"
)
# save model and increment weight decay
print("saving!")
torch.save(model.state_dict(), modelout)
def test(modelin=args.model,outfile=args.out,optimize=args.opt):
# define model, dataloader, 3dmm eigenvectors, optimization method
calib_net = CalibrationNet3(n=1)
sfm_net = CalibrationNet3(n=199)
if modelin != "":
calib_path = os.path.join('model','calib_' + modelin)
sfm_path = os.path.join('model','sfm_' + modelin)
calib_net.load_state_dict(torch.load(calib_path,map_location='cpu'))
sfm_net.load_state_dict(torch.load(sfm_path,map_location='cpu'))
calib_net.to(args.device)
sfm_net.to(args.device)
calib_net.eval()
sfm_net.eval()
# mean shape and eigenvectors for 3dmm
M = 100
data3dmm = dataloader.SyntheticLoader()
mu_lm = torch.from_numpy(data3dmm.mu_lm).float().to(args.device).detach()
mu_lm[:,2] = mu_lm[:,2]*-1
lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float().to(args.device).detach()
sigma = torch.from_numpy(data3dmm.sigma).float().to(args.device).detach()
sigma = torch.diag(sigma.squeeze())
lm_eigenvec = torch.mm(lm_eigenvec, sigma)
batch_size = 10
lm_eigenvec = torch.stack(batch_size*[lm_eigenvec])
# sample from f testing set
allerror_2d = []
allerror_3d = []
allerror_rel3d = []
allerror_relf = []
all_f = []
all_fpred = []
all_depth = []
out_shape = []
out_f = []
seterror_3d = []
seterror_rel3d = []
seterror_relf = []
seterror_2d = []
f_vals = [i*100 for i in range(4,15)]
for f_test in f_vals:
f_test = 1000
# create dataloader
data = dataloader.TestData()
data.batchsize = batch_size
loader = data.createLoader(f_test)
# containers
f_pred = []
shape_pred = []
error_2d = []
error_3d = []
error_rel3d = []
error_relf = []
M = 100;
N = 68;
batch_size = data.batchsize;
for j,data in enumerate(loader):
# load the data
x_cam_gt = data['x_cam_gt'].to(args.device)
shape_gt = data['x_w_gt'].to(args.device)
fgt = data['f_gt'].to(args.device)
x_img = data['x_img'].to(args.device)
x_img_gt = data['x_img_gt'].to(args.device)
T_gt = data['T_gt'].to(args.device)
# reshape and form data
one = torch.ones(batch_size,M*N,1).to(device=args.device)
x_img_one = torch.cat([x_img,one],dim=2)
x_cam_pt = x_cam_gt.permute(0,1,3,2).reshape(batch_size,6800,3)
x = x_img.permute(0,2,1).reshape(batch_size,2,M,N)
ptsI = x_img_one.reshape(batch_size,M,N,3).permute(0,1,3,2)[:,:,:2,:]
# run the model
f = calib_net(x) + 300
betas = sfm_net(x)
betas = betas.squeeze(0).unsqueeze(-1)
shape = mu_lm + torch.bmm(lm_eigenvec,betas).squeeze().view(batch_size,N,3)
# additional optimization on initial solution
if optimize:
calib_net.load_state_dict(torch.load(calib_path,map_location=args.device))
sfm_net.load_state_dict(torch.load(sfm_path,map_location=args.device))
calib_net.train()
sfm_net.train()
opt1 = torch.optim.Adam(calib_net.parameters(),lr=1e-4)
opt2 = torch.optim.Adam(sfm_net.parameters(),lr=1e-2)
curloss = 100
for outerloop in itertools.count():
# camera calibration
shape = shape.detach()
for iter in itertools.count():
opt1.zero_grad()
f = torch.mean(calib_net.forward2(x) + 300)
K = torch.zeros(3,3).float().to(device=args.device)
K[0,0] = f
K[1,1] = f
K[2,2] = 1
# ground truth l1 error
f_error = torch.mean(torch.abs(f - fgt))
# rmse
rmse = torch.norm(shape_gt - shape,dim=2).mean()
# differentiable PnP pose estimation
error1 = []
for i in range(batch_size):
km, c_w, scaled_betas, alphas = util.EPnP(ptsI[i],shape[i],K)
Xc, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape[i],ptsI[i],K)
error2d = util.getReprojError2(ptsI[i],shape[i],R,T,K,show=False,loss='l1')
error1.append(error2d.mean())
# loss
loss = torch.stack(error1).mean()
# stopping condition
if iter == 5: break
if iter > 5 and prev_loss < loss:
break
else:
prev_loss = loss
# update
loss.backward()
opt1.step()
print(f"iter: {iter} | error: {loss.item():.3f} | f/fgt: {f.mean().item():.1f}/{fgt.mean().item():.1f} | error2d: {loss.item():.3f} | rmse: {rmse.item():.3f} ")
# sfm
f = f.detach()
for iter in itertools.count():
opt2.zero_grad()
# shape prediction
betas = sfm_net.forward2(x)
betas = betas.unsqueeze(-1)
shape = mu_lm + torch.bmm(lm_eigenvec,betas).squeeze().view(batch_size,N,3)
K = torch.zeros((3,3)).float()
K[0,0] = f
K[1,1] = f
K[2,2] = 1
#rmse = torch.norm(shape_gt - shape,dim=1).mean().detach()
rmse = torch.norm(shape_gt - shape,dim=2).mean()
# differentiable PnP pose estimation
error1 = []
for i in range(batch_size):
km, c_w, scaled_betas, alphas = util.EPnP(ptsI[i],shape[i],K)
Xc, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape[i],ptsI[i],K)
error2d = util.getReprojError2(ptsI[i],shape[i],R,T,K,show=False,loss='l1')
errorTime = util.getTimeConsistency(shape[i],R,T)
error1.append(error2d.mean())
#loss = torch.stack(error1).mean() + 0.01*torch.stack(error2).mean()
loss = torch.stack(error1).mean()
if iter == 5: break
if iter > 5 and prev_loss < loss:
break
else:
prev_loss = loss
loss.backward()
opt2.step()
print(f"iter: {iter} | error: {loss.item():.3f} | f/fgt: {f.mean().item():.1f}/{fgt.mean().item():.1f} | error2d: {loss.item():.3f} | rmse: {rmse.item():.3f} ")
# closing condition for outerloop on dual objective
if torch.abs(curloss - loss) < 0.01: break
curloss = loss
else:
K = torch.zeros((batch_size,3,3)).float().to(device=args.device)
K[:,0,0] = f.squeeze()
K[:,1,1] = f.squeeze()
K[:,2,2] = 1
km,c_w,scaled_betas,alphas = util.EPnP(ptsI,shape,K)
Xc, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI,K)
#all_fpred.append(batch_size*[f.detach().item()])
e2d,e3d,eshape,e2d_all,e3d_all,d_all = util.getBatchError(ptsI.detach(),shape.detach(),K.detach(),x_cam_gt,shape_gt)
f_error = torch.squeeze(torch.abs(fgt - f)/fgt)
e2d = e2d.cpu().numpy()
e3d = e3d.cpu().numpy()
eshape = eshape.cpu().numpy()
f_error = f_error.cpu().squeeze().numpy()
e2d_all = e2d_all.cpu().numpy()
e3d_all = e3d_all.cpu().numpy()
d_all = d_all.cpu().numpy()
f_pred.append(f.detach().cpu().item())
shape_pred.append(shape.detach().cpu().numpy())
all_depth.append(d_all.flatten())
all_f.append(np.array([fgt.mean()] * d_all.flatten().shape[0]))
all_fpred.append(np.array([f.mean()]*d_all.flatten().shape[0]))
print(f"f/sequence: {f_test}/{j} | f/fgt: {f.mean().item():.3f}/{fgt.mean().item():.3f} | f_error_rel: {f_error.mean().item():.4f} | rmse: {eshape.mean().item():.4f} | rel rmse: {np.mean(e3d):.4f} | 2d error: {np.mean(e2d):.4f}")
avg_2d = np.mean(error_2d)
avg_rel3d = np.mean(error_rel3d)
avg_3d = np.mean(error_3d)
avg_relf = np.mean(error_relf)
seterror_2d.append(avg_2d)
seterror_3d.append(avg_3d)
seterror_rel3d.append(avg_rel3d)
seterror_relf.append(avg_relf)
out_f.append(np.array(f_pred))
out_shape.append(np.concatenate(shape_pred,axis=0))
print(f"f_error_rel: {avg_relf:.4f} | rel rmse: {avg_rel3d:.4f} | 2d error: {avg_2d:.4f} | rmse: {avg_3d:.4f} |")
out_shape = np.stack(out_shape)
out_f = np.stack(out_f)
all_f = np.stack(all_f).flatten()
all_fpred = np.stack(all_fpred).flatten()
all_depth = np.stack(all_depth).flatten()
allerror_2d = np.stack(allerror_2d).flatten()
allerror_rel3d = np.stack(allerror_rel3d).flatten()
matdata = {}
matdata['fvals'] = np.array(f_vals)
matdata['all_f'] = np.array(all_f)
matdata['all_fpred'] = np.array(all_fpred)
matdata['all_d'] = np.array(all_depth)
matdata['error_2d'] = allerror_2d
matdata['error_rel3d'] = allerror_rel3d
matdata['seterror_2d'] = np.array(seterror_2d)
matdata['seterror_3d'] = np.array(seterror_3d)
matdata['seterror_rel3d'] = np.array(seterror_rel3d)
matdata['seterror_relf'] = np.array(seterror_relf)
matdata['out_shape'] = out_shape
matdata['out_f'] = out_f
scipy.io.savemat(outfile,matdata)
print(f"MEAN seterror_2d: {np.mean(seterror_2d)}")
print(f"MEAN seterror_3d: {np.mean(seterror_3d)}")
print(f"MEAN seterror_rel3d: {np.mean(seterror_rel3d)}")
print(f"MEAN seterror_relf: {np.mean(seterror_relf)}")
def train(modelin=args.model,
modelout=args.out,
log=args.log,
logname=args.logname):
# define logger
#torch.manual_seed(6)
# define model, dataloader, 3dmm eigenvectors, optimization method
torch.manual_seed(2)
model = Model1(k=199, feature_transform=False)
if modelin != "":
model.load_state_dict(torch.load(modelin))
model.cuda()
optimizer = torch.optim.Adam(model.parameters(), lr=1e-2)
decay = torch.optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.1)
# dataloader
#data = dataloader.Data()
#loader = data.batchloader
loader = dataloader.SyntheticLoader()
# mean shape and eigenvectors for 3dmm
mu_lm = torch.from_numpy(loader.mu_lm).float().cuda()
#mu_lm[:,2] = mu_lm[:,2] * -1
#shape = mu_lm.detach().cuda()
lm_eigenvec = torch.from_numpy(loader.lm_eigenvec).float().cuda()
M = loader.M
N = loader.N
# main training loop
for epoch in itertools.count():
#for j,batch in enumerate(loader):
np.random.seed(2)
for j, data in enumerate(loader):
# get the input and gt values
x_cam_gt = data['x_cam_gt'].cuda()
x_w_gt = data['x_w_gt'].cuda()
fgt = data['f_gt'].cuda()
beta_gt = data['beta_gt'].cuda()
x_img = data['x_img'].cuda()
#x_img_norm = data['x_img_norm']
x_img_gt = data['x_img_gt'].cuda()
#batch_size = fgt.shape[0]
x_img_pts = x_img.reshape((M, N, 2)).permute(0, 2, 1)
one = torch.ones(M * N, 1).cuda()
x_img_one = torch.cat([x_img, one], dim=1)
x_cam_pt = x_cam_gt.permute(0, 2, 1).reshape(6800, 3)
# run the model
x = x_img_one.permute(1, 0)
# get initial values for betas and alphas of EPNP
ptsI = x_img.reshape((M, N, 2)).permute(0, 2, 1)
v = pptk.viewer([0, 0, 0])
v.set(point_size=1)
# optimize using EPNP+GN
for iter in itertools.count():
optimizer.zero_grad()
# model output
betas, _, _ = model(x.unsqueeze(0))
shape = torch.sum(betas * lm_eigenvec, 1)
shape = shape.reshape(68, 3) + mu_lm
K = torch.zeros((3, 3)).float().cuda()
K[0, 0] = 400
K[1, 1] = 400
K[2, 2] = 1
# differentiable pose estimation
km, c_w, scaled_betas, alphas = util.EPnP(ptsI, shape, K)
Xc, R, T, _ = util.optimizeGN(km, c_w, scaled_betas, alphas,
shape, ptsI, K)
loss = util.getReprojError2(ptsI, shape, R, T, K).mean()
loss.backward()
optimizer.step()
#visualize shape
v.clear()
pts = shape.detach().cpu().numpy()
v.load(pts)
v.set(r=300)
print(
f"iter: {iter} | error: {loss.item():.3f} | f/fgt: {fgt[0].item():.1f}/{fgt[0].item():.1f}"
)
# save model and increment weight decay
print("saving!")
torch.save(model.state_dict(), modelout)
decay.step()
def testBIWI(model,modelin=args.model,outfile=args.out,feature_transform=args.feat_trans):
if modelin != "":
model.load_state_dict(torch.load(modelin))
model.cuda()
# load 3dmm data
data3dmm = dataloader.SyntheticLoader()
mu_lm = torch.from_numpy(data3dmm.mu_lm).float().cuda()
lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float().cuda()
loader = dataloader.BIWILoader()
seterror_3d = []
seterror_rel3d = []
seterror_relf = []
seterror_2d = []
for sub in range(len(loader)):
batch = loader[sub]
x_cam_gt = batch['x_cam_gt'].cuda()
x_w_gt = batch['x_w_gt'].cuda()
f_gt = batch['f_gt'].cuda()
x_img = batch['x_img'].cuda()
x_img_gt = batch['x_img_gt'].cuda()
M = x_img_gt.shape[0]
one = torch.ones(M,1,68).cuda()
x_img_one = torch.cat([x_img,one],dim=1)
# run the model
out, trans, transfeat = model(x_img_one)
alphas = out[:,:199].mean(0)
f = torch.relu(out[:,199]).mean()
K = torch.zeros((3,3)).float().cuda()
K[0,0] = f;
K[1,1] = f;
K[2,2] = 1;
K[0,2] = 320;
K[1,2] = 240;
# apply 3DMM model from predicted parameters
alpha_matrix = torch.diag(alphas)
shape_cov = torch.mm(lm_eigenvec,alpha_matrix)
s = shape_cov.sum(1).view(68,3)
shape = (mu_lm + s)
shae[:,2] = shape[:,2]*-1
# run epnp algorithm
# get control points
c_w = util.getControlPoints(shape)
# solve alphas
alphas = util.solveAlphas(shape,c_w)
# setup M
px = 320;
py = 240;
Matrix = util.setupM(alphas,x_img.permute(0,2,1),px,py,f)
# get eigenvectors of M for each view
u,d,v = torch.svd(Matrix)
#solve N=1
c_c_n1 = v[:,:,-1].reshape((M,4,3)).permute(0,2,1)
_ , x_c_n1, _ = util.scaleControlPoints(c_c_n1,c_w[:3,:],alphas,shape)
Rn1,Tn1 = util.getExtrinsics(x_c_n1,shape)
reproj_error2_n1 = util.getReprojError2(x_img,shape,Rn1,Tn1,K)
reproj_error3_n1 = util.getReprojError3(x_cam_gt,shape,Rn1,Tn1)
rel_error_n1 = util.getRelReprojError3(x_cam_gt,shape,Rn1,Tn1)
# solve N=2
# get distance contraints
d12,d13,d14,d23,d24,d34 = util.getDistances(c_w)
distances = torch.stack([d12,d13,d14,d23,d24,d34])**2
beta_n2 = util.getBetaN2(v[:,:,-2:],distances)
c_c_n2 = util.getControlPointsN2(v[:,:,-2:],beta_n2)
_,x_c_n2,_ = util.scaleControlPoints(c_c_n2,c_w[:3,:],alphas,shape)
Rn2,Tn2 = util.getExtrinsics(x_c_n2,shape)
reproj_error2_n2 = util.getReprojError2(x_img,shape,Rn2,Tn2,K)
reproj_error3_n2 = util.getReprojError3(x_cam_gt,shape,Rn2,Tn2)
rel_error_n2 = util.getRelReprojError3(x_cam_gt,shape,Rn1,Tn1)
mask = reproj_error2_n1 < reproj_error2_n2
reproj_errors = torch.cat((reproj_error2_n1[mask],reproj_error2_n2[~mask]))
rmse_errors = torch.cat((reproj_error3_n1[mask],reproj_error3_n2[~mask]))
rel_errors = torch.cat((rel_error_n2[~mask],rel_error_n1[mask]))
# errors
reproj_error = torch.mean(reproj_errors)
reconstruction_error = torch.mean(rmse_errors)
rel_error = torch.mean(rel_errors)
f_error = torch.abs(f_gt - f) / f_gt
seterror_2d.append(reproj_error.cpu().data.item())
seterror_3d.append(reconstruction_error.cpu().data.item())
seterror_rel3d.append(rel_error.cpu().data.item())
seterror_relf.append(f_error.cpu().data.item())
print(f"fgt: {f_gt.mean().item():.3f} | f_error_rel: {f_error.item():.4f} | rmse: {reconstruction_error.item():.4f} | rel rmse: {rel_error.item():.4f} | 2d error: {reproj_error.item():.4f}")
#end for
matdata = {}
matdata['seterror_2d'] = np.array(seterror_2d)
matdata['seterror_3d'] = np.array(seterror_3d)
matdata['seterror_rel3d'] = np.array(seterror_rel3d)
matdata['seterror_relf'] = np.array(seterror_relf)
scipy.io.savemat(outfile,matdata)
print(f"MEAN seterror_2d: {np.mean(seterror_2d)}")
print(f"MEAN seterror_3d: {np.mean(seterror_3d)}")
print(f"MEAN seterror_rel3d: {np.mean(seterror_rel3d)}")
print(f"MEAN seterror_relf: {np.mean(seterror_relf)}")
def testBIWIID(modelin=args.model,outfile=args.out,optimize=args.opt):
# define model, dataloader, 3dmm eigenvectors, optimization method
calib_net = CalibrationNet3(n=1)
sfm_net = CalibrationNet3(n=199)
if modelin != "":
calib_path = os.path.join('model','calib_' + modelin)
sfm_path = os.path.join('model','sfm_' + modelin)
calib_net.load_state_dict(torch.load(calib_path))
sfm_net.load_state_dict(torch.load(sfm_path))
calib_net.eval()
sfm_net.eval()
# mean shape and eigenvectors for 3dmm
data3dmm = dataloader.SyntheticLoader()
mu_lm = torch.from_numpy(data3dmm.mu_lm).float().detach()
mu_lm[:,2] = mu_lm[:,2]*-1
lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float().detach()
sigma = torch.from_numpy(data3dmm.sigma).float().detach()
sigma = torch.diag(sigma.squeeze())
lm_eigenvec = torch.mm(lm_eigenvec, sigma)
# define loader
loader = dataloader.BIWIIDLoader()
f_pred = []
shape_pred = []
error_2d = []
error_relf = []
error_rel3d = []
for idx in range(len(loader)):
batch = loader[idx]
x_cam_gt = batch['x_cam_gt']
fgt = batch['f_gt']
x_img = batch['x_img']
x_img_gt = batch['x_img_gt']
M = x_img_gt.shape[0]
N = 68
ptsI = x_img.reshape((M,N,2)).permute(0,2,1)
x = ptsI.unsqueeze(0).permute(0,2,1,3)
# run the model
f = calib_net(x) + 300
betas = sfm_net(x)
betas = betas.squeeze(0).unsqueeze(-1)
shape = mu_lm + torch.mm(lm_eigenvec,betas).squeeze().view(N,3)
# additional optimization on initial solution
if optimize:
calib_net.load_state_dict(torch.load(calib_path))
sfm_net.load_state_dict(torch.load(sfm_path))
calib_net.eval()
sfm_net.eval()
trainfc(calib_net)
trainfc(sfm_net)
opt1 = torch.optim.Adam(calib_net.parameters(),lr=1e-4)
opt2 = torch.optim.Adam(sfm_net.parameters(),lr=1e-5)
curloss = 100
for outerloop in itertools.count():
# camera calibration
shape = shape.detach()
for iter in itertools.count():
opt1.zero_grad()
f = calib_net.forward2(x) + 300
K = torch.zeros(3,3).float()
K[0,0] = f
K[1,1] = f
K[2,2] = 1
f_error = torch.mean(torch.abs(f - fgt))
#rmse = torch.norm(shape_gt - shape,dim=1).mean()
# differentiable PnP pose estimation
km,c_w,scaled_betas, alphas = util.EPnP(ptsI,shape,K)
Xc, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI,K)
error2d = util.getReprojError2(ptsI,shape,R,T,K,show=False,loss='l2')
error_time = util.getTimeConsistency(shape,R,T)
loss = error2d.mean() + 0.01*error_time
if iter == 5: break
#if iter > 10 and prev_loss < loss:
# break
#else:
# prev_loss = loss
loss.backward()
opt1.step()
print(f"iter: {iter} | error: {loss.item():.3f} | f/fgt: {f.item():.1f}/{fgt[0].item():.1f} | error2d: {error2d.mean().item():.3f} ")
# sfm
f = f.detach()
for iter in itertools.count():
opt2.zero_grad()
# shape prediction
betas = sfm_net.forward2(x)
shape = torch.sum(betas * lm_eigenvec,1)
shape = shape.reshape(68,3) + mu_lm
shape = shape - shape.mean(0).unsqueeze(0)
K = torch.zeros((3,3)).float()
K[0,0] = f
K[1,1] = f
K[2,2] = 1
#rmse = torch.norm(shape_gt - shape,dim=1).mean().detach()
#rmse = torch.norm(shape_gt - shape,dim=1).mean().detach()
# differentiable PnP pose estimation
km,c_w,scaled_betas,alphas = util.EPnP(ptsI,shape,K)
Xc, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI,K)
error2d = util.getReprojError2(ptsI,shape,R,T,K,show=False,loss='l2')
error_time = util.getTimeConsistency(shape,R,T)
loss = error2d.mean() + 0.01*error_time
if iter == 5: break
prev_loss = loss.item()
loss.backward()
opt2.step()
print(f"iter: {iter} | error: {loss.item():.3f} | f/fgt: {f.item():.1f}/{fgt[0].item():.1f} | error2d: {error2d.mean().item():.3f} ")
# closing condition for outerloop on dual objective
if torch.abs(curloss - loss) < 0.01: break
curloss = loss
else:
K = torch.zeros(3,3).float()
K[0,0] = f
K[1,1] = f
K[2,2] = 1
km,c_w,scaled_betas,alphas = util.EPnP(ptsI,shape,K)
Xc, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI,K)
# get errors
reproj_errors2 = util.getReprojError2(ptsI,shape,R,T,K)
rel_errors = util.getRelReprojError3(x_cam_gt,shape,R,T)
reproj_error = reproj_errors2.mean()
rel_error = rel_errors.mean()
f_error = torch.abs(fgt - f) / fgt
# save final prediction
f_pred.append(f.detach().cpu().item())
shape_pred.append(shape.detach().cpu().numpy())
error_2d.append(reproj_error.cpu().data.item())
error_rel3d.append(rel_error.cpu().data.item())
error_relf.append(f_error.cpu().data.item())
print(f" f/fgt: {f[0].item():.3f}/{fgt.item():.3f} | f_error_rel: {f_error.item():.4f} | rel rmse: {rel_error.item():.4f} | 2d error: {reproj_error.item():.4f}")
#end for
# prepare output file
out_shape = np.stack(shape_pred)
out_f = np.stack(f_pred)
matdata = {}
matdata['shape'] = np.stack(out_shape)
matdata['f'] = np.stack(out_f)
matdata['error_2d'] = np.array(error_2d)
matdata['error_rel3d'] = np.array(error_rel3d)
matdata['error_relf'] = np.array(error_relf)
scipy.io.savemat(outfile,matdata)
print(f"MEAN seterror_2d: {np.mean(error_2d)}")
print(f"MEAN seterror_rel3d: {np.mean(error_rel3d)}")
print(f"MEAN seterror_relf: {np.mean(error_relf)}")
def train(modelin=args.model,
modelout=args.out,
device=args.device,
opt=args.opt,
ft=args.ft):
# define model, dataloader, 3dmm eigenvectors, optimization method
calib_net = PointNet(n=1, feature_transform=ft)
sfm_net = PointNet(n=199, feature_transform=ft)
if modelin != "":
calib_path = os.path.join('model', 'calib_' + modelin)
sfm_path = os.path.join('model', 'sfm_' + modelin)
pretrained1 = torch.load(calib_path)
pretrained2 = torch.load(sfm_path)
calib_dict = calib_net.state_dict()
sfm_dict = sfm_net.state_dict()
pretrained1 = {k: v for k, v in pretrained1.items() if k in calib_dict}
pretrained2 = {k: v for k, v in pretrained2.items() if k in sfm_dict}
calib_dict.update(pretrained1)
sfm_dict.update(pretrained2)
calib_net.load_state_dict(pretrained1)
sfm_net.load_state_dict(pretrained2)
calib_net.to(device=device)
sfm_net.to(device=device)
opt1 = torch.optim.Adam(calib_net.parameters(), lr=1e-3)
opt2 = torch.optim.Adam(sfm_net.parameters(), lr=1e-3)
# dataloader
loader = dataloader.SyntheticLoader()
batch_size = 100
M = loader.M
N = loader.N
# mean shape and eigenvectors for 3dmm
mu_lm = torch.from_numpy(loader.mu_lm).float() #.to(device=device)
mu_lm[:, 2] = mu_lm[:, 2] * -1
mu_lm = torch.stack(batch_size * [mu_lm.to(device=device)])
shape = mu_lm
lm_eigenvec = torch.from_numpy(
loader.lm_eigenvec).float().to(device=device)
sigma = torch.from_numpy(loader.sigma).float().detach().to(device=device)
sigma = torch.diag(sigma.squeeze())
lm_eigenvec = torch.mm(lm_eigenvec, sigma)
lm_eigenvec = torch.stack(M * [lm_eigenvec])
# main training loop
best = 10000
for epoch in itertools.count():
for j, batch in enumerate(loader):
# get the input and gt values
x_cam_gt = batch['x_cam_gt'].to(device=device)
shape_gt = batch['x_w_gt'].to(device=device)
fgt = batch['f_gt'].to(device=device)
x_img = batch['x_img'].to(device=device)
#beta_gt = batch['beta_gt'].to(device=device)
#x_img_norm = batch['x_img_norm']
#x_img_gt = batch['x_img_gt'].to(device=device).permute(0,2,1,3)
x = x_img.reshape(M, N, 2).permute(0, 2, 1)
batch_size = fgt.shape[0]
#x_cam_pt = x_cam_gt.permute(0,1,3,2).reshape(batch_size,6800,3)
#x = x_img.permute(0,2,1).reshape(batch_size,2,M,N)
#ptsI = x_img_one.reshape(batch_size,M,N,3).permute(0,1,3,2)[:,:,:2,:]
# calibration
f = torch.squeeze(calib_net(x) + 300)
K = torch.zeros((M, 3, 3)).float().to(device=device)
K[:, 0, 0] = f
K[:, 1, 1] = f
K[:, 2, 2] = 1
# sfm
betas = sfm_net(x)
betas = betas.unsqueeze(-1)
shape = mu_lm + torch.bmm(lm_eigenvec, betas).squeeze().view(
M, N, 3)
shape = shape - shape.mean(1).unsqueeze(1)
opt1.zero_grad()
opt2.zero_grad()
f_error = torch.mean(torch.abs(f - fgt))
#error2d = torch.mean(torch.abs(pred - x_img_gt))
error3d = torch.mean(torch.norm(shape - shape_gt, dim=2))
error = f_error + error3d
error.backward()
opt1.step()
opt2.step()
print(
f"iter: {j} | best: {best:.2f} | f_error: {f_error.item():.3f} | error3d: {error3d.item():.3f} "
)
if j == 1000: break
# save model and increment weight decay
torch.save(sfm_net.state_dict(), os.path.join('model', 'sfm_model.pt'))
torch.save(calib_net.state_dict(),
os.path.join('model', 'calib_model.pt'))
ferror = test(modelin='model.pt', outfile=args.out, optimize=False)
if ferror < best:
best = ferror
print("saving!")
torch.save(sfm_net.state_dict(),
os.path.join('model', 'sfm_' + modelout))
torch.save(calib_net.state_dict(),
os.path.join('model', 'calib_' + modelout))
sfm_net.train()
calib_net.train()
def test(modelin=args.model,outfile=args.out,optimize=args.opt):
# define model, dataloader, 3dmm eigenvectors, optimization method
calib_net = CalibrationNet3(n=1)
sfm_net = CalibrationNet3(n=199)
if modelin != "":
calib_path = os.path.join('model','calib_' + modelin)
sfm_path = os.path.join('model','sfm_' + modelin)
calib_net.load_state_dict(torch.load(calib_path))
sfm_net.load_state_dict(torch.load(sfm_path))
calib_net.eval()
sfm_net.eval()
# mean shape and eigenvectors for 3dmm
M = 100
data3dmm = dataloader.SyntheticLoader()
mu_lm = torch.from_numpy(data3dmm.mu_lm).float().detach()
mu_lm[:,2] = mu_lm[:,2]*-1
lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float().detach()
sigma = torch.from_numpy(data3dmm.sigma).float().detach()
sigma = torch.diag(sigma.squeeze())
lm_eigenvec = torch.mm(lm_eigenvec, sigma)
# sample from f testing set
allerror_2d = []
allerror_3d = []
allerror_rel3d = []
allerror_relf = []
all_f = []
all_fpred = []
all_depth = []
out_shape = []
out_f = []
seterror_3d = []
seterror_rel3d = []
seterror_relf = []
seterror_2d = []
f_vals = [i*100 for i in range(4,15)]
for f_test in f_vals:
# create dataloader
#f_test = 1000
loader = dataloader.TestLoader(f_test)
f_pred = []
shape_pred = []
error_2d = []
error_3d = []
error_rel3d = []
error_relf = []
M = 100;
N = 68;
batch_size = 1;
for j,data in enumerate(loader):
if j == 10: break
# load the data
x_cam_gt = data['x_cam_gt']
shape_gt = data['x_w_gt']
fgt = data['f_gt']
x_img = data['x_img']
x_img_gt = data['x_img_gt']
T_gt = data['T_gt']
all_depth.append(np.mean(T_gt[:,2]))
all_f.append(fgt.numpy()[0])
ptsI = x_img.reshape((M,N,2)).permute(0,2,1)
x = ptsI.unsqueeze(0).permute(0,2,1,3)
# run the model
f = calib_net(x) + 300
betas = sfm_net(x)
betas = betas.squeeze(0).unsqueeze(-1)
shape = mu_lm + torch.mm(lm_eigenvec,betas).squeeze().view(N,3)
# additional optimization on initial solution
if optimize:
calib_net.load_state_dict(torch.load(calib_path))
sfm_net.load_state_dict(torch.load(sfm_path))
calib_net.eval()
sfm_net.eval()
trainfc(calib_net)
trainfc(sfm_net)
opt1 = torch.optim.Adam(calib_net.parameters(),lr=1e-4)
opt2 = torch.optim.Adam(sfm_net.parameters(),lr=1e-2)
curloss = 100
for outerloop in itertools.count():
# camera calibration
shape = shape.detach()
for iter in itertools.count():
opt1.zero_grad()
f = calib_net.forward2(x) + 300
K = torch.zeros(3,3).float()
K[0,0] = f
K[1,1] = f
K[2,2] = 1
f_error = torch.mean(torch.abs(f - fgt))
rmse = torch.norm(shape_gt - shape,dim=1).mean()
# differentiable PnP pose estimation
km,c_w,scaled_betas, alphas = util.EPnP(ptsI,shape,K)
Xc, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI,K)
error2d = util.getReprojError2(ptsI,shape,R,T,K,show=False,loss='l2')
#error2d = util.getReprojError2_(ptsI,Xc,K,show=True,loss='l2')
error_time = util.getTimeConsistency(shape,R,T)
loss = error2d.mean() + 0.01*error_time
if iter == 5: break
loss.backward()
opt1.step()
print(f"iter: {iter} | error: {loss.item():.3f} | f/fgt: {f.item():.1f}/{fgt[0].item():.1f} | error2d: {error2d.mean().item():.3f} | rmse: {rmse.item():.3f} ")
# sfm
f = f.detach()
for iter in itertools.count():
opt2.zero_grad()
# shape prediction
betas = sfm_net.forward2(x)
shape = torch.sum(betas * lm_eigenvec,1)
shape = shape.reshape(68,3) + mu_lm
shape = shape - shape.mean(0).unsqueeze(0)
K = torch.zeros((3,3)).float()
K[0,0] = f
K[1,1] = f
K[2,2] = 1
#rmse = torch.norm(shape_gt - shape,dim=1).mean().detach()
rmse = torch.norm(shape_gt - shape,dim=1).mean().detach()
# differentiable PnP pose estimation
km,c_w,scaled_betas,alphas = util.EPnP(ptsI,shape,K)
Xc, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI,K)
error2d = util.getReprojError2(ptsI,shape,R,T,K,show=False,loss='l2')
error_time = util.getTimeConsistency(shape,R,T)
loss = error2d.mean() + 0.01*error_time
if iter == 5: break
if iter > 10 and prev_loss < loss:
break
else:
prev_loss = loss
loss.backward()
opt2.step()
print(f"iter: {iter} | error: {loss.item():.3f} | f/fgt: {f.item():.1f}/{fgt[0].item():.1f} | error2d: {error2d.mean().item():.3f} | rmse: {rmse.item():.3f} ")
# closing condition for outerloop on dual objective
if torch.abs(curloss - loss) < 0.01: break
curloss = loss
else:
K = torch.zeros(3,3).float()
K[0,0] = f
K[1,1] = f
K[2,2] = 1
km,c_w,scaled_betas,alphas = util.EPnP(ptsI,shape,K)
Xc, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI,K)
all_fpred.append(f.detach().numpy()[0])
# get errors
reproj_errors2 = util.getReprojError2(ptsI,shape,R,T,K,show=False)
reproj_errors3 = torch.norm(shape_gt - shape,dim=1).mean()
rel_errors = util.getRelReprojError3(x_cam_gt,shape,R,T)
reproj_error = reproj_errors2.mean()
reconstruction_error = reproj_errors3.mean()
rel_error = rel_errors.mean()
f_error = torch.abs(fgt - f) / fgt
# save final prediction
f_pred.append(f.detach().cpu().item())
shape_pred.append(shape.detach().cpu().numpy())
allerror_3d.append(reproj_error.data.numpy())
allerror_2d.append(reconstruction_error.data.numpy())
allerror_rel3d.append(rel_error.data.numpy())
error_2d.append(reproj_error.cpu().data.item())
error_3d.append(reconstruction_error.cpu().data.item())
error_rel3d.append(rel_error.cpu().data.item())
error_relf.append(f_error.cpu().data.item())
print(f"f/sequence: {f_test}/{j} | f/fgt: {f[0].item():.3f}/{fgt.item():.3f} | f_error_rel: {f_error.item():.4f} | rmse: {reconstruction_error.item():.4f} | rel rmse: {rel_error.item():.4f} | 2d error: {reproj_error.item():.4f}")
avg_2d = np.mean(error_2d)
avg_rel3d = np.mean(error_rel3d)
avg_3d = np.mean(error_3d)
avg_relf = np.mean(error_relf)
seterror_2d.append(avg_2d)
seterror_3d.append(avg_3d)
seterror_rel3d.append(avg_rel3d)
seterror_relf.append(avg_relf)
out_f.append(np.stack(f_pred))
out_shape.append(np.stack(shape_pred,axis=0))
print(f"f_error_rel: {avg_relf:.4f} | rel rmse: {avg_rel3d:.4f} | 2d error: {reproj_error.item():.4f} | rmse: {avg_3d:.4f} |")
out_shape = np.stack(out_shape)
out_f = np.stack(out_f)
all_f = np.stack(all_f).flatten()
all_fpred = np.stack(all_fpred).flatten()
all_d = np.stack(all_depth).flatten()
allerror_2d = np.stack(allerror_2d).flatten()
allerror_3d = np.stack(allerror_3d).flatten()
allerror_rel3d = np.stack(allerror_rel3d).flatten()
matdata = {}
matdata['fvals'] = np.array(f_vals)
matdata['all_f'] = np.array(all_f)
matdata['all_fpred'] = np.array(all_fpred)
matdata['all_d'] = np.array(all_depth)
matdata['error_2d'] = allerror_2d
matdata['error_3d'] = allerror_3d
matdata['error_rel3d'] = allerror_rel3d
matdata['seterror_2d'] = np.array(seterror_2d)
matdata['seterror_3d'] = np.array(seterror_3d)
matdata['seterror_rel3d'] = np.array(seterror_rel3d)
matdata['seterror_relf'] = np.array(seterror_relf)
matdata['shape'] = np.stack(out_shape)
matdata['f'] = np.stack(out_f)
scipy.io.savemat(outfile,matdata)
print(f"MEAN seterror_2d: {np.mean(seterror_2d)}")
print(f"MEAN seterror_3d: {np.mean(seterror_3d)}")
print(f"MEAN seterror_rel3d: {np.mean(seterror_rel3d)}")
print(f"MEAN seterror_relf: {np.mean(seterror_relf)}")
def test(modelin=args.model,
outfile=args.out,
feature_transform=args.feat_trans):
# define model, dataloader, 3dmm eigenvectors, optimization method
#if modelin != "":
# model.load_state_dict(torch.load(modelin))
#model.eval()
#model.cuda()
# mean shape and eigenvectors for 3dmm
M = 100
N = 68
data3dmm = dataloader.SyntheticLoader()
mu_lm = torch.from_numpy(data3dmm.mu_lm).float()
mu_lm[:, 2] = mu_lm[:, 2] * -1
shape = mu_lm.detach()
lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float()
#optimizer = torch.optim.Adam(model.parameters(),lr=1e-2)
# sample from f testing set
allerror_2d = []
allerror_3d = []
allerror_rel3d = []
allerror_relf = []
all_f = []
all_depth = []
seterror_3d = []
seterror_rel3d = []
seterror_relf = []
seterror_2d = []
f_vals = [i * 100 for i in range(4, 21)]
np.random.seed(0)
for f_test in f_vals:
f_test = 1200
# create dataloader
loader = dataloader.TestLoader(f_test)
error_2d = []
error_3d = []
error_rel3d = []
error_relf = []
M = 100
N = 68
batch_size = 1
for j, data in enumerate(loader):
# create a model and optimizer for it
#model2 = Model1(k=199,feature_transform=False)
#model2.apply(util.init_weights)
model = Model1(k=1, feature_transform=False)
model.apply(util.init_weights)
optimizer = torch.optim.Adam(model.parameters(), lr=2e-1)
#data = loader[67]
x_cam_gt = data['x_cam_gt']
shape = data['x_w_gt']
fgt = data['f_gt']
x_img = data['x_img']
x_img_gt = data['x_img_gt']
T_gt = data['T_gt']
all_depth.append(np.mean(T_gt[:, 2]))
all_f.append(fgt.numpy()[0])
x_img_pts = x_img.reshape((M, N, 2)).permute(0, 2, 1)
one = torch.ones(M * N, 1)
x_img_one = torch.cat([x_img, one], dim=1)
x_cam_pt = x_cam_gt.permute(0, 2, 1).reshape(M * N, 3)
x = x_img_one.permute(1, 0)
ptsI = x_img.reshape((M, N, 2)).permute(0, 2, 1)
for iter in itertools.count():
optimizer.zero_grad()
#betas,_,_ = model2(x.unsqueeze(0))
#shape = torch.sum(betas * lm_eigenvec,1)
#shape = shape.reshape(68,3) + mu_lm
f, _, _ = model(x.unsqueeze(0))
#f = f + 300
#f = (torch.nn.functional.tanh(f)+1)*850 + 300
f = f + 300
#f = torch.nn.functional.sigmoid(f)
K = torch.zeros((3, 3)).float()
K[0, 0] = f
K[1, 1] = f
K[2, 2] = 1
# differentiable pose estimation
km, c_w, scaled_betas, alphas = util.EPnP(ptsI, shape, K)
Xc, R, T, mask = util.optimizeGN(km, c_w, scaled_betas, alphas,
shape, ptsI, K)
error2d = util.getReprojError2(ptsI,
shape,
R,
T,
K,
show=False,
loss='l1')
loss = error2d.mean()
loss.backward()
if torch.any(model.fc2.weight.grad != model.fc2.weight.grad):
print("oh oh something broke")
break
optimizer.step()
print(
f"iter: {iter} | error: {loss.item():.3f} | f/fgt: {f.item():.1f}/{fgt[0].item():.1f}"
)
if iter == 200: break
# get errors
reproj_errors2 = util.getReprojError2(ptsI, shape, R, T, K)
reproj_errors3 = util.getReprojError3(x_cam_gt, shape, R, T)
rel_errors = util.getRelReprojError3(x_cam_gt, shape, R, T)
reproj_error = reproj_errors2.mean()
reconstruction_error = reproj_errors3.mean()
rel_error = rel_errors.mean()
f_error = torch.abs(fgt - f) / fgt
allerror_3d.append(reproj_error.data.numpy())
allerror_2d.append(reconstruction_error.data.numpy())
allerror_rel3d.append(rel_error.data.numpy())
error_2d.append(reproj_error.cpu().data.item())
error_3d.append(reconstruction_error.cpu().data.item())
error_rel3d.append(rel_error.cpu().data.item())
error_relf.append(f_error.cpu().data.item())
print(
f"f/sequence: {f_test}/{j} | f/fgt: {f[0].item():.3f}/{fgt.item():.3f} | f_error_rel: {f_error.item():.4f} | rmse: {reconstruction_error.item():.4f} | rel rmse: {rel_error.item():.4f} | 2d error: {reproj_error.item():.4f}"
)
#end for
avg_2d = np.mean(error_2d)
avg_rel3d = np.mean(error_rel3d)
avg_3d = np.mean(error_3d)
avg_relf = np.mean(error_relf)
seterror_2d.append(avg_2d)
seterror_3d.append(avg_3d)
seterror_rel3d.append(avg_rel3d)
seterror_relf.append(avg_relf)
#end for
break
all_f = np.stack(all_f).flatten()
all_d = np.stack(all_depth).flatten()
allerror_2d = np.stack(allerror_2d).flatten()
allerror_3d = np.stack(allerror_3d).flatten()
allerror_rel3d = np.stack(allerror_rel3d).flatten()
matdata = {}
matdata['fvals'] = np.array(f_vals)
matdata['all_f'] = np.array(all_f)
matdata['all_d'] = np.array(all_depth)
matdata['error_2d'] = allerror_2d
matdata['error_3d'] = allerror_3d
matdata['error_rel3d'] = allerror_rel3d
matdata['seterror_2d'] = np.array(seterror_2d)
matdata['seterror_3d'] = np.array(seterror_3d)
matdata['seterror_rel3d'] = np.array(seterror_rel3d)
matdata['seterror_relf'] = np.array(seterror_relf)
scipy.io.savemat(outfile, matdata)
print(f"MEAN seterror_2d: {np.mean(seterror_2d)}")
print(f"MEAN seterror_3d: {np.mean(seterror_3d)}")
print(f"MEAN seterror_rel3d: {np.mean(seterror_rel3d)}")
print(f"MEAN seterror_relf: {np.mean(seterror_relf)}")
def test_sfm(modelin=args.model, outfile=args.out, optimize=args.opt):
# define model, dataloader, 3dmm eigenvectors, optimization method
calib_net = CalibrationNet3(n=1)
sfm_net = CalibrationNet3(n=199)
calib_path = os.path.join('model', 'calib_' + modelin)
sfm_path = os.path.join('model', 'sfm_' + modelin)
# mean shape and eigenvectors for 3dmm
M = 100
data3dmm = dataloader.SyntheticLoader()
mu_lm = torch.from_numpy(data3dmm.mu_lm).float().detach()
mu_lm[:, 2] = mu_lm[:, 2] * -1
lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float().detach()
sigma = torch.from_numpy(data3dmm.sigma).float().detach()
sigma = torch.diag(sigma.squeeze())
lm_eigenvec = torch.mm(lm_eigenvec, sigma)
# sample from f testing set
allerror_2d = []
allerror_3d = []
allerror_rel3d = []
allerror_relf = []
all_f = []
all_fpred = []
all_depth = []
out_shape = []
out_f = []
seterror_3d = []
seterror_rel3d = []
seterror_relf = []
seterror_2d = []
f_vals = [i * 100 for i in range(4, 15)]
for f_test in f_vals:
# create dataloader
#f_test = 1000
loader = dataloader.TestLoader(f_test)
f_pred = []
shape_pred = []
error_2d = []
error_3d = []
error_rel3d = []
error_relf = []
M = 100
N = 68
batch_size = 1
training_pred = np.zeros((10, 100, 68, 3))
training_gt = np.zeros((10, 100, 68, 3))
for j, data in enumerate(loader):
if j == 10: break
# load the data
x_cam_gt = data['x_cam_gt']
shape_gt = data['x_w_gt']
fgt = data['f_gt']
x_img = data['x_img']
x_img_gt = data['x_img_gt']
T_gt = data['T_gt']
all_depth.append(np.mean(T_gt[:, 2]))
all_f.append(fgt.numpy()[0])
ptsI = x_img.reshape((M, N, 2)).permute(0, 2, 1)
x = ptsI.unsqueeze(0).permute(0, 2, 1, 3)
# test camera calibration
#calib_net.load_state_dict(torch.load(calib_path))
opt2 = torch.optim.Adam(sfm_net.parameters(), lr=1e-5)
sfm_net.eval()
trainfc(sfm_net)
f = 2000
for iter in itertools.count():
opt2.zero_grad()
# shape prediction
betas = sfm_net.forward2(x)
betas = torch.clamp(betas, -20, 20)
shape = torch.sum(betas * lm_eigenvec, 1)
shape = shape.reshape(68, 3) + mu_lm
shape = shape - shape.mean(0).unsqueeze(0)
rmse = torch.norm(shape_gt - shape, dim=1).mean().detach()
K = torch.zeros((3, 3)).float()
K[0, 0] = f
K[1, 1] = f
K[2, 2] = 1
km, c_w, scaled_betas, alphas = util.EPnP(ptsI, shape, K)
Xc, R, T, mask = util.optimizeGN(km, c_w, scaled_betas, alphas,
shape, ptsI, K)
error2d = util.getReprojError2(ptsI,
shape,
R,
T,
K,
show=False,
loss='l2')
error_time = util.getTimeConsistency(shape, R, T)
loss = error2d.mean() + 0.01 * error_time
loss.backward()
opt2.step()
print(
f"iter: {iter} | error: {loss.item():.3f} | error2d: {error2d.mean().item():.3f} | rmse: {rmse.item():.3f} "
)
if iter == 100: break
training_pred[j, iter, :, :] = shape.detach().cpu().numpy()
training_gt[j, iter, :, :] = shape_gt.detach().cpu().numpy()
# get errors
reproj_errors2 = util.getReprojError2(ptsI,
shape,
R,
T,
K,
show=False)
reproj_errors3 = torch.norm(shape_gt - shape, dim=1).mean()
rel_errors = util.getRelReprojError3(x_cam_gt, shape, R, T)
reproj_error = reproj_errors2.mean()
reconstruction_error = reproj_errors3.mean()
rel_error = rel_errors.mean()
f_error = torch.abs(fgt - f) / fgt
# save final prediction
shape_pred.append(shape.detach().cpu().numpy())
allerror_3d.append(reproj_error.data.numpy())
allerror_2d.append(reconstruction_error.data.numpy())
allerror_rel3d.append(rel_error.data.numpy())
error_2d.append(reproj_error.cpu().data.item())
error_3d.append(reconstruction_error.cpu().data.item())
error_rel3d.append(rel_error.cpu().data.item())
error_relf.append(f_error.cpu().data.item())
print(
f"f/sequence: {f_test}/{j} | f/fgt: {f:.3f}/{fgt.item():.3f} | f_error_rel: {f_error.item():.4f} | rmse: {reconstruction_error.item():.4f} | rel rmse: {rel_error.item():.4f} | 2d error: {reproj_error.item():.4f}"
)
avg_2d = np.mean(error_2d)
avg_rel3d = np.mean(error_rel3d)
avg_3d = np.mean(error_3d)
avg_relf = np.mean(error_relf)
seterror_2d.append(avg_2d)
seterror_3d.append(avg_3d)
seterror_rel3d.append(avg_rel3d)
seterror_relf.append(avg_relf)
out_shape.append(np.stack(shape_pred, axis=0))
print(
f"f_error_rel: {avg_relf:.4f} | rel rmse: {avg_rel3d:.4f} | 2d error: {reproj_error.item():.4f} | rmse: {avg_3d:.4f} |"
)
all_f = np.stack(all_f).flatten()
all_d = np.stack(all_depth).flatten()
allerror_2d = np.stack(allerror_2d).flatten()
allerror_3d = np.stack(allerror_3d).flatten()
allerror_rel3d = np.stack(allerror_rel3d).flatten()
matdata = {}
matdata['training_pred'] = training_pred
matdata['training_gt'] = training_gt
matdata['fvals'] = np.array(f_vals)
matdata['all_f'] = np.array(all_f)
matdata['all_d'] = np.array(all_depth)
matdata['error_2d'] = allerror_2d
matdata['error_3d'] = allerror_3d
matdata['error_rel3d'] = allerror_rel3d
matdata['seterror_2d'] = np.array(seterror_2d)
matdata['seterror_3d'] = np.array(seterror_3d)
matdata['seterror_rel3d'] = np.array(seterror_rel3d)
matdata['seterror_relf'] = np.array(seterror_relf)
scipy.io.savemat(outfile, matdata)
print(f"MEAN seterror_2d: {np.mean(seterror_2d)}")
print(f"MEAN seterror_3d: {np.mean(seterror_3d)}")
print(f"MEAN seterror_rel3d: {np.mean(seterror_rel3d)}")
print(f"MEAN seterror_relf: {np.mean(seterror_relf)}")
def test(modelin=args.model,
outfile=args.out,
feature_transform=args.feat_trans):
# define model, dataloader, 3dmm eigenvectors, optimization method
#if modelin != "":
# model.load_state_dict(torch.load(modelin))
#model.eval()
#model.cuda()
# mean shape and eigenvectors for 3dmm
M = 100
N = 68
data3dmm = dataloader.SyntheticLoader()
mu_lm = torch.from_numpy(data3dmm.mu_lm).float()
mu_lm[:, 2] = mu_lm[:, 2] * -1
shape = mu_lm.detach()
lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float()
#optimizer = torch.optim.Adam(model.parameters(),lr=1e-2)
# sample from f testing set
allerror_2d = []
allerror_3d = []
allerror_rel3d = []
allerror_relf = []
all_f = []
all_depth = []
seterror_3d = []
seterror_rel3d = []
seterror_relf = []
seterror_2d = []
f_vals = [i * 100 for i in range(4, 21)]
for f_test in f_vals:
f_test = 1400
# create dataloader
loader = dataloader.TestLoader(f_test)
error_2d = []
error_3d = []
error_rel3d = []
error_relf = []
M = 100
N = 68
batch_size = 1
for j, data in enumerate(loader):
# create a model and optimizer for it
model = Model2(k=1, feature_transform=False)
model.apply(util.init_weights)
optimizer = torch.optim.Adam(model.parameters(), lr=1e-1)
M = loader.M
N = loader.N
# load the data
T_gt = data['T_gt']
x_cam_gt = data['x_cam_gt']
x_w_gt = data['x_w_gt']
fgt = data['f_gt']
x_img = data['x_img']
x_img_gt = data['x_img_gt']
x_img_pts = x_img.reshape((M, N, 2)).permute(0, 2, 1)
one = torch.ones(M * N, 1)
x_img_one = torch.cat([x_img, one], dim=1)
x_cam_pt = x_cam_gt.permute(0, 2, 1).reshape(M * N, 3)
all_depth.append(np.mean(T_gt[:, 2]))
all_f.append(fgt.numpy()[0])
# create the input
b = 10
x = x_img_one.reshape(M, N,
3).reshape(b, M // b, N,
3).reshape(b, M // b * N, 3)
x = x.permute(0, 2, 1)
ptsI = x_img.reshape((M, N, 2)).permute(0, 2, 1)
# optimize using EPNP+GN
fvals = []
errors = []
for iter in itertools.count():
optimizer.zero_grad()
f, _, _ = model(x)
#f = f + 1000
f = torch.nn.functional.leaky_relu(f) + 300
K = torch.zeros((b, 3, 3)).float()
K[:, 0, 0] = f.squeeze()
K[:, 1, 1] = f.squeeze()
K[:, 2, 2] = 1
# differentiable pose estimation
losses = []
for i in range(b):
j = i + 1
km, c_w, scaled_betas, alphas = util.EPnP(
ptsI[i:j * b], shape, K[i])
Xc, R, T, _ = util.optimizeGN(km, c_w, scaled_betas,
alphas, shape, ptsI[i:j * b],
K[i])
error2d = util.getReprojError2(ptsI[i:j * b], shape, R, T,
K[i]).mean()
losses.append(error2d)
loss = torch.stack(losses).mean()
loss.backward()
optimizer.step()
print(
f"iter: {iter} | error: {loss.item():.3f} | f/fgt: {f.mean().item():.1f}/{fgt[0].item():.1f}"
)
if iter == 100: break
# get overall poses
f = f.mean()
K = torch.zeros((3, 3)).float()
K[0, 0] = f
K[1, 1] = f
K[2, 2] = 1
km, c_w, scaled_betas, alphas = util.EPnP(ptsI, shape, K)
Xc, R, T, _ = util.optimizeGN(km, c_w, scaled_betas, alphas, shape,
ptsI, K)
# get errors
reproj_errors2 = util.getReprojError2(ptsI, shape, R, T, K)
reproj_errors3 = util.getReprojError3(x_cam_gt, shape, R, T)
rel_errors = util.getRelReprojError3(x_cam_gt, shape, R, T)
reproj_error = reproj_errors2.mean()
reconstruction_error = reproj_errors3.mean()
rel_error = rel_errors.mean()
f_error = torch.abs(fgt - f) / fgt
allerror_3d.append(reproj_error.data.numpy())
allerror_2d.append(reconstruction_error.data.numpy())
allerror_rel3d.append(rel_error.data.numpy())
error_2d.append(reproj_error.cpu().data.item())
error_3d.append(reconstruction_error.cpu().data.item())
error_rel3d.append(rel_error.cpu().data.item())
error_relf.append(f_error.cpu().data.item())
print(
f"f/sequence: {f_test}/{j} | f/fgt: {f.item():.3f}/{fgt.item():.3f} | f_error_rel: {f_error.item():.4f} | rmse: {reconstruction_error.item():.4f} | rel rmse: {rel_error.item():.4f} | 2d error: {reproj_error.item():.4f}"
)
#end for
avg_2d = np.mean(error_2d)
avg_rel3d = np.mean(error_rel3d)
avg_3d = np.mean(error_3d)
avg_relf = np.mean(error_relf)
seterror_2d.append(avg_2d)
seterror_3d.append(avg_3d)
seterror_rel3d.append(avg_rel3d)
seterror_relf.append(avg_relf)
#end for
break
all_f = np.stack(all_f).flatten()
all_d = np.stack(all_depth).flatten()
allerror_2d = np.stack(allerror_2d).flatten()
allerror_3d = np.stack(allerror_3d).flatten()
allerror_rel3d = np.stack(allerror_rel3d).flatten()
matdata = {}
matdata['fvals'] = np.array(f_vals)
matdata['all_f'] = np.array(all_f)
matdata['all_d'] = np.array(all_depth)
matdata['error_2d'] = allerror_2d
matdata['error_3d'] = allerror_3d
matdata['error_rel3d'] = allerror_rel3d
matdata['seterror_2d'] = np.array(seterror_2d)
matdata['seterror_3d'] = np.array(seterror_3d)
matdata['seterror_rel3d'] = np.array(seterror_rel3d)
matdata['seterror_relf'] = np.array(seterror_relf)
scipy.io.savemat(outfile, matdata)
print(f"MEAN seterror_2d: {np.mean(seterror_2d)}")
print(f"MEAN seterror_3d: {np.mean(seterror_3d)}")
print(f"MEAN seterror_rel3d: {np.mean(seterror_rel3d)}")
print(f"MEAN seterror_relf: {np.mean(seterror_relf)}")
parser = argparse.ArgumentParser(description="training arguments")
parser.add_argument("--model", default="net.pt")
parser.add_argument("--out",default="results/exp.mat")
parser.add_argument("--device",default='cpu')
parser.add_argument("--opt", default=False, action='store_true')
parser.add_argument("--db", default="syn")
parser.add_argument("--ft",default=False, action="store_true")
args = parser.parse_args()
np.random.seed(0)
torch.manual_seed(0)
##############################################################################################
##############################################################################################
##############################################################################################
data3dmm = dataloader.SyntheticLoader()
mu_lm = torch.from_numpy(data3dmm.mu_lm).float().detach()
mu_lm[:,2] = mu_lm[:,2]*-1
lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float().detach()
sigma = torch.from_numpy(data3dmm.sigma).float().detach()
sigma = torch.diag(sigma.squeeze())
lm_eigenvec = torch.mm(lm_eigenvec, sigma)
# HELPER FUNCTIONS
def trainfc(model):
for name, param in model.named_parameters():
if 'fc' in name and 'feat' not in name:
param.requires_grad = True
def getLoader(db):
if db == 'syn':
def test(modelin=args.model,outfile=args.out,feature_transform=args.feat_trans):
# define model, dataloader, 3dmm eigenvectors, optimization method
#if modelin != "":
# model.load_state_dict(torch.load(modelin))
#model.eval()
#model.cuda()
# mean shape and eigenvectors for 3dmm
M = 100
N = 68
data3dmm = dataloader.SyntheticLoader()
mu_lm = torch.from_numpy(data3dmm.mu_lm).float()
mu_lm[:,2] = mu_lm[:,2]*-1
lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float()
#optimizer = torch.optim.Adam(model.parameters(),lr=1e-2)
# sample from f testing set
allerror_2d = []
allerror_3d = []
allerror_rel3d = []
allerror_relf = []
all_f = []
all_depth = []
seterror_3d = []
seterror_rel3d = []
seterror_relf = []
seterror_2d = []
f_vals = [400 + i*100 for i in range(4)]
# set random seed for reproducibility of test set
np.random.seed(0)
for f_test in f_vals:
f_test = 1400
# create dataloader
loader = dataloader.TestLoader(f_test)
error_2d = []
error_3d = []
error_rel3d = []
error_relf = []
M = 100;
N = 68;
batch_size = 1;
for j, data in enumerate(loader):
# create a model and optimizer for it
theta1 = (1.1*torch.randn(4)).requires_grad()
optimizer = torch.optim.SGD({theta},lr=0.00001)
model2 = Model1(k=199,feature_transform=False)
model2.apply(util.init_weights)
model = Model1(k=1, feature_transform=False)
model.apply(util.init_weights)
optimizer = torch.optim.Adam(list(model.parameters()) + list(model2.parameters()),lr=1)
# load the data
x_cam_gt = data['x_cam_gt']
shape_gt = data['x_w_gt']
fgt = data['f_gt']
x_img = data['x_img']
x_img_gt = data['x_img_gt']
T_gt = data['T_gt']
all_depth.append(np.mean(T_gt[:,2]))
all_f.append(fgt.numpy()[0])
ptsI = x_img.reshape((M,N,2)).permute(0,2,1)
x2d = x_img.view((M,N,2))
x_img_pts = x_img.reshape((M,N,2)).permute(0,2,1)
one = torch.ones(M*N,1)
x_img_one = torch.cat([x_img,one],dim=1)
x = x_img_one.permute(1,0)
ini_pose = torch.zeros((M,6))
ini_pose[:,5] = 99
pre_loss = 99
for iter in itertools.count():
optimizer.zero_grad()
# shape prediction
betas,_,_ = model2(x.unsqueeze(0))
shape = torch.sum(betas * lm_eigenvec,1)
shape = shape.reshape(68,3) + mu_lm
#shape = shape_gt
# RMSE between GT and predicted shape
rmse = torch.norm(shape_gt - shape,dim=1).mean().detach()
# focal length prediction
f,_,_ = model(x.unsqueeze(0))
f = f + 300
K = torch.zeros((3,3)).float()
K[0,0] = f
K[1,1] = f
K[2,2] = 1
# differentiable PnP pose estimation
pose = bpnp(x2d,shape,K,ini_pose)
pred = BPnP.batch_project(pose,shape,K)
# loss
#loss = torch.mean(torch.abs(pred - x2d))
loss = torch.mean(torch.norm(pred - x2d,dim=2))
loss.backward()
optimizer.step()
print(f"iter: {iter} | error: {loss.item():.3f} | f/fgt: {f.item():.1f}/{fgt[0].item():.1f} | rmse: {rmse.item():.2f}")
if iter == 200: break
ini_pose = pose.detach()
# get errors
km,c_w,scaled_betas, alphas = util.EPnP(ptsI,shape,K)
Xc, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI,K)
# get errors
reproj_errors2 = util.getReprojError2(ptsI,shape,R,T,K)
reproj_errors3 = util.getReprojError3(x_cam_gt,shape,R,T)
rel_errors = util.getRelReprojError3(x_cam_gt,shape,R,T)
reproj_error = reproj_errors2.mean()
reconstruction_error = reproj_errors3.mean()
rel_error = rel_errors.mean()
f_error = torch.abs(fgt - f) / fgt
allerror_3d.append(reproj_error.data.numpy())
allerror_2d.append(reconstruction_error.data.numpy())
allerror_rel3d.append(rel_error.data.numpy())
error_2d.append(reproj_error.cpu().data.item())
error_3d.append(reconstruction_error.cpu().data.item())
error_rel3d.append(rel_error.cpu().data.item())
error_relf.append(f_error.cpu().data.item())
print(f"f/sequence: {f_test}/{j} | f/fgt: {f[0].item():.3f}/{fgt.item():.3f} | f_error_rel: {f_error.item():.4f} | rmse: {reconstruction_error.item():.4f} | rel rmse: {rel_error.item():.4f} | 2d error: {reproj_error.item():.4f}")
#end for
avg_2d = np.mean(error_2d)
avg_rel3d = np.mean(error_rel3d)
avg_3d = np.mean(error_3d)
avg_relf = np.mean(error_relf)
seterror_2d.append(avg_2d)
seterror_3d.append(avg_3d)
seterror_rel3d.append(avg_rel3d)
seterror_relf.append(avg_relf)
#end for
break
all_f = np.stack(all_f).flatten()
all_d = np.stack(all_depth).flatten()
allerror_2d = np.stack(allerror_2d).flatten()
allerror_3d = np.stack(allerror_3d).flatten()
allerror_rel3d = np.stack(allerror_rel3d).flatten()
matdata = {}
matdata['fvals'] = np.array(f_vals)
matdata['all_f'] = np.array(all_f)
matdata['all_d'] = np.array(all_depth)
matdata['error_2d'] = allerror_2d
matdata['error_3d'] = allerror_3d
matdata['error_rel3d'] = allerror_rel3d
matdata['seterror_2d'] = np.array(seterror_2d)
matdata['seterror_3d'] = np.array(seterror_3d)
matdata['seterror_rel3d'] = np.array(seterror_rel3d)
matdata['seterror_relf'] = np.array(seterror_relf)
scipy.io.savemat(outfile,matdata)
print(f"MEAN seterror_2d: {np.mean(seterror_2d)}")
print(f"MEAN seterror_3d: {np.mean(seterror_3d)}")
print(f"MEAN seterror_rel3d: {np.mean(seterror_rel3d)}")
print(f"MEAN seterror_relf: {np.mean(seterror_relf)}")
def test(modelin=args.model,outfile=args.out,optimize=args.opt,ft=args.ft):
# define model, dataloader, 3dmm eigenvectors, optimization method
calib_net = PointNet(n=1,feature_transform=ft)
sfm_net = PointNet(n=199,feature_transform=ft)
if modelin != "":
calib_path = os.path.join('model','calib_' + modelin)
sfm_path = os.path.join('model','sfm_' + modelin)
calib_net.load_state_dict(torch.load(calib_path))
sfm_net.load_state_dict(torch.load(sfm_path))
calib_net.eval()
sfm_net.eval()
# mean shape and eigenvectors for 3dmm
M = 100
data3dmm = dataloader.SyntheticLoader()
mu_lm = torch.from_numpy(data3dmm.mu_lm).float().detach()
mu_lm[:,2] = mu_lm[:,2]*-1
lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float().detach()
sigma = torch.from_numpy(data3dmm.sigma).float().detach()
sigma = torch.diag(sigma.squeeze())
lm_eigenvec = torch.mm(lm_eigenvec, sigma)
# sample from f testing set
allerror_2d = []
allerror_3d = []
allerror_rel3d = []
allerror_relf = []
all_f = []
all_fpred = []
all_depth = []
out_shape = []
out_f = []
seterror_3d = []
seterror_rel3d = []
seterror_relf = []
seterror_2d = []
f_vals = [i*100 for i in range(4,15)]
# set random seed for reproducibility of test set
np.random.seed(0)
torch.manual_seed(0)
for f_test in f_vals:
# create dataloader
loader = dataloader.TestLoader(f_test)
f_pred = []
shape_pred = []
error_2d = []
error_3d = []
error_rel3d = []
error_relf = []
M = 100;
N = 68;
batch_size = 1;
for j,data in enumerate(loader):
if j >= 10: break
# load the data
x_cam_gt = data['x_cam_gt']
shape_gt = data['x_w_gt']
fgt = data['f_gt']
x_img = data['x_img']
x_img_gt = data['x_img_gt']
depth = torch.norm(x_cam_gt.mean(2),dim=1)
all_depth.append(depth.numpy())
all_f.append(fgt.numpy()[0])
ptsI = x_img.reshape((M,N,2)).permute(0,2,1)
x = x_img.unsqueeze(0).permute(0,2,1)
# run the model
f = calib_net(x) + 300
betas = sfm_net(x)
betas = betas.squeeze(0).unsqueeze(-1)
shape = mu_lm + torch.mm(lm_eigenvec,betas).squeeze().view(N,3)
shape = shape - shape.mean(0).unsqueeze(0)
# get motion measurement guess
K = torch.zeros((3,3)).float()
K[0,0] = f
K[1,1] = f
K[2,2] = 1
km,c_w,scaled_betas,alphas = util.EPnP(ptsI,shape,K)
_, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI)
error_time = util.getTimeConsistency(shape,R,T)
if error_time > 10:
mode='walk'
else:
mode='still'
# apply dual optimization
if optimize:
calib_net.load_state_dict(torch.load(calib_path))
sfm_net.load_state_dict(torch.load(sfm_path))
shape,K,R,T = dualoptimization(x,calib_net,sfm_net,shape_gt=shape_gt,fgt=fgt,mode=mode)
f = K[0,0].detach()
else:
K = torch.zeros(3,3).float()
K[0,0] = f
K[1,1] = f
K[2,2] = 1
km,c_w,scaled_betas,alphas = util.EPnP(ptsI,shape,K)
Xc, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI)
# get errors
reproj_errors2 = util.getReprojError2(ptsI,shape,R,T,K,show=False)
reproj_errors3 = torch.norm(shape_gt - shape,dim=1).mean()
rel_errors = util.getRelReprojError3(x_cam_gt,shape,R,T)
reproj_error = reproj_errors2.mean()
reconstruction_error = reproj_errors3.mean()
rel_error = rel_errors.mean()
f_error = torch.abs(fgt - f) / fgt
# save final prediction
f_pred.append(f.detach().cpu().item())
shape_pred.append(shape.detach().cpu().numpy())
all_fpred.append(f.detach().data.numpy())
allerror_3d.append(reproj_error.data.numpy())
allerror_2d.append(reconstruction_error.data.numpy())
allerror_rel3d.append(rel_error.data.numpy())
error_2d.append(reproj_error.cpu().data.item())
error_3d.append(reconstruction_error.cpu().data.item())
error_rel3d.append(rel_error.cpu().data.item())
error_relf.append(f_error.cpu().data.item())
print(f"f/sequence: {f_test}/{j} | f/fgt: {f.item():.3f}/{fgt.item():.3f} | f_error_rel: {f_error.item():.4f} | rmse: {reconstruction_error.item():.4f} | rel rmse: {rel_error.item():.4f} | 2d error: {reproj_error.item():.4f}")
avg_2d = np.mean(error_2d)
avg_rel3d = np.mean(error_rel3d)
avg_3d = np.mean(error_3d)
avg_relf = np.mean(error_relf)
seterror_2d.append(avg_2d)
seterror_3d.append(avg_3d)
seterror_rel3d.append(avg_rel3d)
seterror_relf.append(avg_relf)
out_f.append(np.stack(f_pred))
out_shape.append(np.concatenate(shape_pred,axis=0))
print(f"f_error_rel: {avg_relf:.4f} | rel rmse: {avg_rel3d:.4f} | 2d error: {avg_2d:.4f} | rmse: {avg_3d:.4f} |")
# save output
out_shape = np.stack(out_shape)
out_f = np.stack(out_f)
all_f = np.stack(all_f).flatten()
all_fpred = np.stack(all_fpred).flatten()
all_d = np.stack(all_depth).flatten()
allerror_2d = np.stack(allerror_2d).flatten()
allerror_rel3d = np.stack(allerror_rel3d).flatten()
matdata = {}
matdata['fvals'] = np.array(f_vals)
matdata['all_f'] = np.array(all_f)
matdata['all_fpred'] = np.array(all_fpred)
matdata['all_d'] = np.array(all_depth)
matdata['error_2d'] = allerror_2d
matdata['error_3d'] = allerror_3d
matdata['error_rel3d'] = allerror_rel3d
matdata['seterror_2d'] = np.array(seterror_2d)
matdata['seterror_3d'] = np.array(seterror_3d)
matdata['seterror_rel3d'] = np.array(seterror_rel3d)
matdata['seterror_relf'] = np.array(seterror_relf)
matdata['shape'] = np.stack(out_shape)
matdata['f'] = np.stack(out_f)
scipy.io.savemat(outfile,matdata)
print(f"MEAN seterror_2d: {np.mean(seterror_2d)}")
print(f"MEAN seterror_3d: {np.mean(seterror_3d)}")
print(f"MEAN seterror_rel3d: {np.mean(seterror_rel3d)}")
print(f"MEAN seterror_relf: {np.mean(seterror_relf)}")
return np.mean(seterror_relf)
def test(model,
modelin=args.model,
outfile=args.out,
feature_transform=args.feat_trans):
# define model, dataloader, 3dmm eigenvectors, optimization method
if modelin != "":
model.load_state_dict(torch.load(modelin))
model.eval()
# mean shape and eigenvectors for 3dmm
M = 100
data3dmm = dataloader.SyntheticLoader()
mu_lm = torch.from_numpy(data3dmm.mu_lm).float()
lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float()
shape = mu_lm
# sample from f testing set
allerror_2d = []
allerror_3d = []
allerror_rel3d = []
allerror_relf = []
all_f = []
all_depth = []
seterror_3d = []
seterror_rel3d = []
seterror_relf = []
seterror_2d = []
f_vals = [i * 100 for i in range(4, 21)]
for f_test in f_vals:
# create dataloader
data = dataloader.TestLoader(f_test)
error_2d = []
error_3d = []
error_rel3d = []
error_relf = []
M = 100
N = 68
batch_size = 1
for k in range(len(data)):
batch = data[4]
x_cam_gt = batch['x_cam_gt']
x_w_gt = batch['x_w_gt']
f_gt = batch['f_gt']
x_img = batch['x_img'].unsqueeze(0)
x_img_gt = batch['x_img_gt']
T_gt = batch['T_gt']
sequence = batch['x_img'].reshape((M, N, 2)).permute(0, 2, 1)
all_depth.append(np.mean(T_gt[:, 2]))
all_f.append(f_gt.numpy()[0])
x = x_img.reshape((batch_size, M, N, 2)).permute(0, 3, 2, 1) / 640
x_one = torch.cat(
[x.squeeze().permute(2, 0, 1) * 640,
torch.ones(M, 1, N)],
dim=1)
# run the model
out = model(x)
betas = out[:, :199]
fout = torch.relu(out[:, 199])
if torch.any(fout < 1): fout = fout + 1
# apply 3DMM model from predicted parameters
alpha_matrix = torch.diag(betas.squeeze())
shape_cov = torch.mm(lm_eigenvec, alpha_matrix)
s = shape_cov.sum(1).view(68, 3)
#shape = (mu_lm + s)
#shape = mu_lm
#shape[:,2] = shape[:,2]*-1
# create variables and optimizer for variables as SGD
# run epnp using predicted shape and intrinsics
varf = Variable(fout, requires_grad=True)
K = torch.zeros((3, 3))
K[0, 0] = varf
K[1, 1] = varf
K[2, 2] = 1
K[0, 2] = 0
K[1, 2] = 0
Xc, R, T = util.EPnP(sequence, shape, K)
tmpT = T.detach()
tmpR = R.detach()
varR = Variable(R, requires_grad=True)
varT = Variable(T, requires_grad=True)
optimizer = torch.optim.Adam([varR, varT], lr=1e-1)
# optimize results for image consistency
ferror = []
losses = []
minerror = 10000
for iter in itertools.count():
K = torch.zeros((3, 3))
K[0, 0] = varf
K[1, 1] = varf
K[2, 2] = 1
K[0, 2] = 0
K[1, 2] = 0
R = varR
T = varT
Xc, _, _ = util.EPnP(sequence, shape, K)
#Xc,R,T = util.EPnP(sequence,shape,K)
optimizer.zero_grad()
# k inverse
kinv = torch.zeros(3, 3).float()
kinv[0, 0] = 1 / varf
kinv[1, 1] = 1 / varf
kinv[2, 2] = 1
# get errors
reproj_errors2 = util.getReprojError2(sequence, shape, R, T, K)
#reproj_errors3 = util.getReprojError3(x_cam_gt,shape,varR,varT)
error_3d = util.getRelReprojError3(x_cam_gt, shape, R,
T).mean()
#error_3d = util.getPCError(x_cam_gt,x_one.permute(0,2,1),torch.stack(100*[kinv]),mode='l2')
error_Rconsistency = util.getRConsistency(R)
error_Tconsistency = util.getTConsistency(T) * 0.001
error_3dconsistency = util.get3DConsistency(
sequence, shape, kinv, R, T)
reproj_error = torch.mean(reproj_errors2)
# determine convergence
loss = error_3dconsistency
if loss < minerror:
minerror = loss
minf = varf.item()
minR = R
minT = T
convergence = 0
else:
convergence += 1
loss.backward()
optimizer.step()
f = util.solvef(sequence, Xc.detach())
print(f)
#if varf < 0: varf = varf*-1
delta = K[0, 0] - varf
direction = torch.sign(delta)
error_f = torch.abs(varf - f_gt) / f_gt
ferror.append(error_f.item())
losses.append(loss.item())
print(
f"iter: {iter} | loss: {loss.item():.3f} | f/fgt: {varf.item():.3f}/{f_gt.item():.3f} | 2d error: {reproj_error.item():.3f} | error R: {error_Rconsistency.item():.3f} | error T: {error_Tconsistency.item():.3f} | error 3d: {error_3dconsistency.item():.3f} | GT RMSE: {error_3d.item():.3f} | delta: {delta.item():.3f}"
)
if convergence == 100: break
data = {'ferror': np.array(ferror), 'loss': np.array(losses)}
scipy.io.savemat("optimizationlr1.mat", data)
quit()
reconstruction_error = reproj_errors3.mean()
rel_error = rel_errors.mean()
f_error = torch.abs(f_gt - fout) / f_gt
allerror_3d.append(reproj_error.data.numpy())
allerror_2d.append(reconstruction_error.data.numpy())
allerror_rel3d.append(rel_error.data.numpy())
error_2d.append(reproj_error.cpu().data.item())
error_3d.append(reconstruction_error.cpu().data.item())
error_rel3d.append(rel_error.cpu().data.item())
error_relf.append(f_error.cpu().data.item())
print(
f"f/sequence: {f_test}/{k} | f/fgt: {fout[0].item():.3f}/{f_gt.item():.3f} | f_error_rel: {f_error.item():.4f} | rmse: {reconstruction_error.item():.4f} | rel rmse: {rel_error.item():.4f} | 2d error: {reproj_error.item():.4f}"
)
#end for
avg_2d = np.mean(error_2d)
avg_rel3d = np.mean(error_rel3d)
avg_3d = np.mean(error_3d)
avg_relf = np.mean(error_relf)
seterror_2d.append(avg_2d)
seterror_3d.append(avg_3d)
seterror_rel3d.append(avg_rel3d)
seterror_relf.append(avg_relf)
#end for
all_f = np.stack(all_f).flatten()
all_d = np.stack(all_depth).flatten()
allerror_2d = np.stack(allerror_2d).flatten()
allerror_3d = np.stack(allerror_3d).flatten()
allerror_rel3d = np.stack(allerror_rel3d).flatten()
matdata = {}
matdata['fvals'] = np.array(f_vals)
matdata['all_f'] = np.array(all_f)
matdata['all_d'] = np.array(all_depth)
matdata['error_2d'] = allerror_2d
matdata['error_3d'] = allerror_3d
matdata['error_rel3d'] = allerror_rel3d
matdata['seterror_2d'] = np.array(seterror_2d)
matdata['seterror_3d'] = np.array(seterror_3d)
matdata['seterror_rel3d'] = np.array(seterror_rel3d)
matdata['seterror_relf'] = np.array(seterror_relf)
scipy.io.savemat(outfile, matdata)
print(f"MEAN seterror_2d: {np.mean(seterror_2d)}")
print(f"MEAN seterror_3d: {np.mean(seterror_3d)}")
print(f"MEAN seterror_rel3d: {np.mean(seterror_rel3d)}")
print(f"MEAN seterror_relf: {np.mean(seterror_relf)}")
def test(model, modelin=args.model,outfile=args.out,feature_transform=args.feat_trans):
# define model, dataloader, 3dmm eigenvectors, optimization method
if modelin != "":
model.load_state_dict(torch.load(modelin))
model.eval()
# mean shape and eigenvectors for 3dmm
M = 100
data3dmm = dataloader.SyntheticLoader()
mu_lm = torch.from_numpy(data3dmm.mu_lm).float()
lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float()
shape = mu_lm
# sample from f testing set
allerror_2d = []
allerror_3d = []
allerror_rel3d = []
allerror_relf = []
all_f = []
all_depth = []
seterror_3d = []
seterror_rel3d = []
seterror_relf = []
seterror_2d = []
f_vals = [i*100 for i in range(4,21)]
for f_test in f_vals:
# create dataloader
data = dataloader.TestLoader(f_test)
error_2d = []
error_3d = []
error_rel3d = []
error_relf = []
M = 100;
N = 68;
batch_size = 1;
for k in range(len(data)):
batch = data[k]
x_cam_gt = batch['x_cam_gt']
x_w_gt = batch['x_w_gt']
f_gt = batch['f_gt']
x_img = batch['x_img'].unsqueeze(0)
x_img_gt = batch['x_img_gt']
T_gt = batch['T_gt']
sequence = batch['x_img'].reshape((M,N,2)).permute(0,2,1)
all_depth.append(np.mean(T_gt[:,2]))
all_f.append(f_gt.numpy()[0])
one = torch.ones(batch_size,M*N,1)
x_img_one = torch.cat([x_img,one],dim=2)
# run the model
out,_,_ = model(x_img_one.permute(0,2,1))
betas = out[:,:199]
fout = torch.relu(out[:,199])
if torch.any(fout < 1): fout = fout+1
# apply 3DMM model from predicted parameters
alpha_matrix = torch.diag(betas.squeeze())
shape_cov = torch.mm(lm_eigenvec,alpha_matrix)
s = shape_cov.sum(1).view(68,3)
#shape = (mu_lm + s)
#shape = mu_lm
#shape[:,2] = shape[:,2]*-1
# run epnp using predicted shape and intrinsics
K = torch.zeros((3,3))
K[0,0] = fout;
K[1,1] = fout;
K[2,2] = 1;
K[0,2] = 0;
K[1,2] = 0;
Xc,R,T = util.EPnP(sequence,shape,K)
# get errors
reproj_errors2 = util.getReprojError2(sequence,shape,R,T,K)
reproj_errors3 = util.getReprojError3(x_cam_gt,shape,R,T)
rel_errors = util.getRelReprojError3(x_cam_gt,shape,R,T)
reproj_error = reproj_errors2.mean()
reconstruction_error = reproj_errors3.mean()
rel_error = rel_errors.mean()
f_error = torch.abs(f_gt - fout) / f_gt
allerror_3d.append(reproj_error.data.numpy())
allerror_2d.append(reconstruction_error.data.numpy())
allerror_rel3d.append(rel_error.data.numpy())
error_2d.append(reproj_error.cpu().data.item())
error_3d.append(reconstruction_error.cpu().data.item())
error_rel3d.append(rel_error.cpu().data.item())
error_relf.append(f_error.cpu().data.item())
print(f"f/sequence: {f_test}/{k} | f/fgt: {fout[0].item():.3f}/{f_gt.item():.3f} | f_error_rel: {f_error.item():.4f} | rmse: {reconstruction_error.item():.4f} | rel rmse: {rel_error.item():.4f} | 2d error: {reproj_error.item():.4f}")
#end for
avg_2d = np.mean(error_2d)
avg_rel3d = np.mean(error_rel3d)
avg_3d = np.mean(error_3d)
avg_relf = np.mean(error_relf)
seterror_2d.append(avg_2d)
seterror_3d.append(avg_3d)
seterror_rel3d.append(avg_rel3d)
seterror_relf.append(avg_relf)
#end for
all_f = np.stack(all_f).flatten()
all_d = np.stack(all_depth).flatten()
allerror_2d = np.stack(allerror_2d).flatten()
allerror_3d = np.stack(allerror_3d).flatten()
allerror_rel3d = np.stack(allerror_rel3d).flatten()
matdata = {}
matdata['fvals'] = np.array(f_vals)
matdata['all_f'] = np.array(all_f)
matdata['all_d'] = np.array(all_depth)
matdata['error_2d'] = allerror_2d
matdata['error_3d'] = allerror_3d
matdata['error_rel3d'] = allerror_rel3d
matdata['seterror_2d'] = np.array(seterror_2d)
matdata['seterror_3d'] = np.array(seterror_3d)
matdata['seterror_rel3d'] = np.array(seterror_rel3d)
matdata['seterror_relf'] = np.array(seterror_relf)
scipy.io.savemat(outfile,matdata)
print(f"MEAN seterror_2d: {np.mean(seterror_2d)}")
print(f"MEAN seterror_3d: {np.mean(seterror_3d)}")
print(f"MEAN seterror_rel3d: {np.mean(seterror_rel3d)}")
print(f"MEAN seterror_relf: {np.mean(seterror_relf)}")
def test(modelin=args.model,
outfile=args.out,
feature_transform=args.feat_trans):
# define model, dataloader, 3dmm eigenvectors, optimization method
#if modelin != "":
# model.load_state_dict(torch.load(modelin))
#model.eval()
#model.cuda()
# mean shape and eigenvectors for 3dmm
M = 100
N = 68
data3dmm = dataloader.SyntheticLoader()
mu_lm = torch.from_numpy(data3dmm.mu_lm).float().detach()
mu_lm[:, 2] = mu_lm[:, 2] * -1
lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float()
#optimizer = torch.optim.Adam(model.parameters(),lr=1e-2)
# sample from f testing set
allerror_2d = []
allerror_3d = []
allerror_rel3d = []
allerror_relf = []
all_f = []
all_depth = []
seterror_3d = []
seterror_rel3d = []
seterror_relf = []
seterror_2d = []
f_vals = [400 + i * 100 for i in range(4)]
# set random seed for reproducibility of test set
np.random.seed(0)
torch.manual_seed(0)
for f_test in f_vals:
f_test = 1400
# create dataloader
loader = dataloader.TestLoader(f_test)
error_2d = []
error_3d = []
error_rel3d = []
error_relf = []
M = 100
N = 68
batch_size = 1
for j, data in enumerate(loader):
# create a model and optimizer for it
model2 = Model1(k=199, feature_transform=False)
model2.apply(util.init_weights)
model = Model1(k=1, feature_transform=False)
model.apply(util.init_weights)
opt1 = torch.optim.Adam(model2.parameters(), lr=1e-1)
opt2 = torch.optim.Adam(model.parameters(), lr=1e-1)
# load the data
x_cam_gt = data['x_cam_gt']
shape_gt = data['x_w_gt']
fgt = data['f_gt']
x_img = data['x_img']
x_img_gt = data['x_img_gt']
T_gt = data['T_gt']
all_depth.append(np.mean(T_gt[:, 2]))
all_f.append(fgt.numpy()[0])
x_img_pts = x_img.reshape((M, N, 2)).permute(0, 2, 1)
one = torch.ones(M * N, 1)
x_img_one = torch.cat([x_img, one], dim=1)
x_cam_pt = x_cam_gt.permute(0, 2, 1).reshape(M * N, 3)
x = x_img_one.permute(1, 0)
ptsI = x_img.reshape((M, N, 2)).permute(0, 2, 1)
# multi objective optimization
shape = mu_lm
for outerloop in itertools.count():
# calibration alg3
shape = shape.detach()
for iter2 in itertools.count():
opt2.zero_grad()
# focal length prediction
curf, _, _ = model(x.unsqueeze(0))
curf = curf + 300
K = torch.zeros((3, 3)).float()
K[0, 0] = curf
K[1, 1] = curf
K[2, 2] = 1
# RMSE between GT and predicted shape
rmse = torch.norm(shape_gt - shape, dim=1).mean().detach()
# differentiable PnP pose estimation
km, c_w, scaled_betas, alphas = util.EPnP(ptsI, shape, K)
Xc, R, T, mask = util.optimizeGN(km, c_w, scaled_betas,
alphas, shape, ptsI, K)
error2d = util.getReprojError2(ptsI,
shape,
R,
T,
K,
show=False,
loss='l2')
loss = error2d.mean()
if iter2 > 20 and prev_loss < loss:
break
else:
prev_loss = loss
loss.backward()
opt2.step()
print(
f"iter: {iter2} | error: {loss.item():.3f} | f/fgt: {curf.item():.1f}/{fgt[0].item():.1f} | rmse: {rmse.item():.2f}"
)
# sfm alg2
curf = curf.detach()
for iter1 in itertools.count():
opt1.zero_grad()
# shape prediction
betas, _, _ = model2(x.unsqueeze(0))
shape = torch.sum(betas * lm_eigenvec, 1)
shape = shape.reshape(68, 3) + mu_lm
K = torch.zeros((3, 3)).float()
K[0, 0] = curf
K[1, 1] = curf
K[2, 2] = 1
# RMSE between GT and predicted shape
rmse = torch.norm(shape_gt - shape, dim=1).mean().detach()
# differentiable PnP pose estimation
km, c_w, scaled_betas, alphas = util.EPnP(ptsI, shape, K)
Xc, R, T, mask = util.optimizeGN(km, c_w, scaled_betas,
alphas, shape, ptsI, K)
error2d = util.getReprojError2(ptsI,
shape,
R,
T,
K,
show=False,
loss='l2')
loss = error2d.mean()
if iter1 > 20 and prev_loss < loss:
break
else:
prev_loss = loss
loss.backward()
opt1.step()
print(
f"iter: {iter1} | error: {loss.item():.3f} | f/fgt: {curf.item():.1f}/{fgt[0].item():.1f} | rmse: {rmse.item():.2f}"
)
# closing condition for outerloop on dual objective
if outerloop == 4: break
f = curf
# get errors
reproj_errors2 = util.getReprojError2(ptsI, shape, R, T, K)
reproj_errors3 = util.getReprojError3(x_cam_gt, shape, R, T)
rel_errors = util.getRelReprojError3(x_cam_gt, shape, R, T)
reproj_error = reproj_errors2.mean()
reconstruction_error = reproj_errors3.mean()
rel_error = rel_errors.mean()
f_error = torch.abs(fgt - f) / fgt
allerror_3d.append(reproj_error.data.numpy())
allerror_2d.append(reconstruction_error.data.numpy())
allerror_rel3d.append(rel_error.data.numpy())
error_2d.append(reproj_error.cpu().data.item())
error_3d.append(reconstruction_error.cpu().data.item())
error_rel3d.append(rel_error.cpu().data.item())
error_relf.append(f_error.cpu().data.item())
print(
f"f/sequence: {f_test}/{j} | f/fgt: {f[0].item():.3f}/{fgt.item():.3f} | f_error_rel: {f_error.item():.4f} | rmse: {reconstruction_error.item():.4f} | rel rmse: {rel_error.item():.4f} | 2d error: {reproj_error.item():.4f}"
)
#end for
avg_2d = np.mean(error_2d)
avg_rel3d = np.mean(error_rel3d)
avg_3d = np.mean(error_3d)
avg_relf = np.mean(error_relf)
seterror_2d.append(avg_2d)
seterror_3d.append(avg_3d)
seterror_rel3d.append(avg_rel3d)
seterror_relf.append(avg_relf)
#end for
break
all_f = np.stack(all_f).flatten()
all_d = np.stack(all_depth).flatten()
allerror_2d = np.stack(allerror_2d).flatten()
allerror_3d = np.stack(allerror_3d).flatten()
allerror_rel3d = np.stack(allerror_rel3d).flatten()
matdata = {}
matdata['fvals'] = np.array(f_vals)
matdata['all_f'] = np.array(all_f)
matdata['all_d'] = np.array(all_depth)
matdata['error_2d'] = allerror_2d
matdata['error_3d'] = allerror_3d
matdata['error_rel3d'] = allerror_rel3d
matdata['seterror_2d'] = np.array(seterror_2d)
matdata['seterror_3d'] = np.array(seterror_3d)
matdata['seterror_rel3d'] = np.array(seterror_rel3d)
matdata['seterror_relf'] = np.array(seterror_relf)
scipy.io.savemat(outfile, matdata)
print(f"MEAN seterror_2d: {np.mean(seterror_2d)}")
print(f"MEAN seterror_3d: {np.mean(seterror_3d)}")
print(f"MEAN seterror_rel3d: {np.mean(seterror_rel3d)}")
print(f"MEAN seterror_relf: {np.mean(seterror_relf)}")