def getLoader(db):
if db == 'syn':
loader = dataloader.TestLoader(f_test)
elif db == 'human36':
loader = dataloader.Human36Loader()
elif db == 'cad120':
loader = dataloader.Cad120Loader()
elif db == 'biwi':
loader = dataloader.BIWILoader()
elif db == 'biwiid':
loader = dataloader.BIWIIDLoader()
return loader
def testBIWI(model,modelin=args.model,outfile=args.out,feature_transform=args.feat_trans):
if modelin != "":
model.load_state_dict(torch.load(modelin))
model.eval()
# load 3dmm data
data3dmm = dataloader.SyntheticLoader()
mu_lm = torch.from_numpy(data3dmm.mu_lm).float()
lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float()
shape = mu_lm
shape[:,2] = shape[:,2] * -1
loader = dataloader.BIWILoader()
seterror_3d = []
seterror_rel3d = []
seterror_relf = []
seterror_2d = []
for sub in range(len(loader)):
batch = loader[sub]
x_cam_gt = batch['x_cam_gt']
x_w_gt = batch['x_w_gt']
f_gt = batch['f_gt']
x_img = batch['x_img']
x_img_gt = batch['x_img_gt']
M = x_img_gt.shape[0]
one = torch.ones(M,1,68)
x_img_one = torch.cat([x_img,one],dim=1)
# run the model
out, trans, transfeat = model(x_img_one)
alphas = out[:,:199].mean(0)
f = torch.relu(out[:,199]).mean()
K = torch.zeros((3,3)).float()
K[0,0] = f;
K[1,1] = f;
K[2,2] = 1;
K[0,2] = 320;
K[1,2] = 240;
Xc,R,T = util.EPnP(x_img,shape,K)
# apply 3DMM model from predicted parameters
reproj_errors2 = util.getReprojError2(x_img,shape,R,T,K)
reproj_errors3 = util.getReprojError3(x_cam_gt,shape,R,T)
rel_errors = util.getRelReprojError3(x_cam_gt,shape,R,T)
reproj_error = reproj_errors2.mean()
reconstruction_error = reproj_errors3.mean()
rel_error = rel_errors.mean()
f_error = torch.abs(f_gt - f) / f_gt
seterror_2d.append(reproj_error.cpu().data.item())
seterror_3d.append(reconstruction_error.cpu().data.item())
seterror_rel3d.append(rel_error.cpu().data.item())
seterror_relf.append(f_error.cpu().data.item())
print(f"fgt: {f_gt.mean().item():.3f} | f_error_rel: {f_error.item():.4f} | rmse: {reconstruction_error.item():.4f} | rel rmse: {rel_error.item():.4f} | 2d error: {reproj_error.item():.4f}")
#end for
matdata = {}
matdata['seterror_2d'] = np.array(seterror_2d)
matdata['seterror_3d'] = np.array(seterror_3d)
matdata['seterror_rel3d'] = np.array(seterror_rel3d)
matdata['seterror_relf'] = np.array(seterror_relf)
scipy.io.savemat(outfile,matdata)
print(f"MEAN seterror_2d: {np.mean(seterror_2d)}")
print(f"MEAN seterror_3d: {np.mean(seterror_3d)}")
print(f"MEAN seterror_rel3d: {np.mean(seterror_rel3d)}")
print(f"MEAN seterror_relf: {np.mean(seterror_relf)}")
def testBIWI(modelin=args.model,outfile=args.out,optimize=args.opt):
# define model, dataloader, 3dmm eigenvectors, optimization method
calib_net = CalibrationNet3(n=1)
sfm_net = CalibrationNet3(n=199)
if modelin != "":
calib_path = os.path.join('model','calib_' + modelin)
sfm_path = os.path.join('model','sfm_' + modelin)
calib_net.load_state_dict(torch.load(calib_path))
sfm_net.load_state_dict(torch.load(sfm_path))
calib_net.eval()
sfm_net.eval()
# mean shape and eigenvectors for 3dmm
data3dmm = dataloader.SyntheticLoader()
mu_lm = torch.from_numpy(data3dmm.mu_lm).float().detach()
mu_lm[:,2] = mu_lm[:,2]*-1
lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float().detach()
sigma = torch.from_numpy(data3dmm.sigma).float().detach()
sigma = torch.diag(sigma.squeeze())
lm_eigenvec = torch.mm(lm_eigenvec, sigma)
# define loader
loader = dataloader.BIWILoader()
f_pred = []
shape_pred = []
error_2d = []
error_relf = []
error_rel3d = []
for sub in range(len(loader)):
batch = loader[sub]
x_cam_gt = batch['x_cam_gt']
fgt = batch['f_gt']
x_img = batch['x_img']
x_img_gt = batch['x_img_gt']
M = x_img_gt.shape[0]
N = 68
ptsI = x_img.reshape((M,N,2)).permute(0,2,1)
x = ptsI.unsqueeze(0).permute(0,2,1,3)
# run the model
f = calib_net(x) + 300
betas = sfm_net(x)
betas = betas.squeeze(0).unsqueeze(-1)
shape = mu_lm + torch.mm(lm_eigenvec,betas).squeeze().view(N,3)
# additional optimization on initial solution
if optimize:
calib_net.load_state_dict(torch.load(calib_path))
sfm_net.load_state_dict(torch.load(sfm_path))
calib_net.eval()
sfm_net.eval()
trainfc(calib_net)
trainfc(sfm_net)
opt1 = torch.optim.Adam(calib_net.parameters(),lr=1e-4)
opt2 = torch.optim.Adam(sfm_net.parameters(),lr=1e-5)
curloss = 100
for outerloop in itertools.count():
# camera calibration
shape = shape.detach()
for iter in itertools.count():
opt1.zero_grad()
f = calib_net.forward2(x) + 300
K = torch.zeros(3,3).float()
K[0,0] = f
K[1,1] = f
K[2,2] = 1
f_error = torch.mean(torch.abs(f - fgt))
#rmse = torch.norm(shape_gt - shape,dim=1).mean()
# differentiable PnP pose estimation
km,c_w,scaled_betas, alphas = util.EPnP(ptsI,shape,K)
Xc, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI,K)
error2d = util.getReprojError2(ptsI,shape,R,T,K,show=False,loss='l2')
error_time = util.getTimeConsistency(shape,R,T)
loss = error2d.mean() + 0.01*error_time
if iter == 5: break
if iter > 10 and prev_loss < loss:
break
else:
prev_loss = loss
loss.backward()
opt1.step()
print(f"iter: {iter} | error: {loss.item():.3f} | f/fgt: {f.item():.1f}/{fgt[0].item():.1f} | error2d: {error2d.mean().item():.3f} ")
# sfm
f = f.detach()
for iter in itertools.count():
opt2.zero_grad()
# shape prediction
betas = sfm_net.forward2(x)
shape = torch.sum(betas * lm_eigenvec,1)
shape = shape.reshape(68,3) + mu_lm
shape = shape - shape.mean(0).unsqueeze(0)
K = torch.zeros((3,3)).float()
K[0,0] = f
K[1,1] = f
K[2,2] = 1
#rmse = torch.norm(shape_gt - shape,dim=1).mean().detach()
#rmse = torch.norm(shape_gt - shape,dim=1).mean().detach()
# differentiable PnP pose estimation
km,c_w,scaled_betas,alphas = util.EPnP(ptsI,shape,K)
Xc, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI,K)
error2d = util.getReprojError2(ptsI,shape,R,T,K,show=False,loss='l2')
error_time = util.getTimeConsistency(shape,R,T)
loss = error2d.mean() + 0.01*error_time
if iter == 5: break
if iter > 10 and prev_loss < loss:
break
else:
prev_loss = loss
loss.backward()
opt2.step()
print(f"iter: {iter} | error: {loss.item():.3f} | f/fgt: {f.item():.1f}/{fgt[0].item():.1f} | error2d: {error2d.mean().item():.3f} ")
# closing condition for outerloop on dual objective
if torch.abs(curloss - loss) < 0.01: break
curloss = loss
else:
K = torch.zeros(3,3).float()
K[0,0] = f
K[1,1] = f
K[2,2] = 1
km,c_w,scaled_betas,alphas = util.EPnP(ptsI,shape,K)
Xc, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI,K)
# get errors
reproj_errors2 = util.getReprojError2(ptsI,shape,R,T,K)
rel_errors = util.getRelReprojError3(x_cam_gt,shape,R,T)
reproj_error = reproj_errors2.mean()
rel_error = rel_errors.mean()
f_error = torch.abs(fgt - f) / fgt
# save final prediction
f_pred.append(f.detach().cpu().item())
shape_pred.append(shape.detach().cpu().numpy())
error_2d.append(reproj_error.cpu().data.item())
error_rel3d.append(rel_error.cpu().data.item())
error_relf.append(f_error.cpu().data.item())
print(f" f/fgt: {f[0].item():.3f}/{fgt.item():.3f} | f_error_rel: {f_error.item():.4f} | rel rmse: {rel_error.item():.4f} | 2d error: {reproj_error.item():.4f}")
#end for
# prepare output file
out_shape = np.stack(shape_pred)
out_f = np.stack(f_pred)
matdata = {}
matdata['shape'] = np.stack(out_shape)
matdata['f'] = np.stack(out_f)
matdata['error_2d'] = np.array(error_2d)
matdata['error_rel3d'] = np.array(error_rel3d)
matdata['error_relf'] = np.array(error_relf)
scipy.io.savemat(outfile,matdata)
print(f"MEAN seterror_2d: {np.mean(error_2d)}")
print(f"MEAN seterror_rel3d: {np.mean(error_rel3d)}")
print(f"MEAN seterror_relf: {np.mean(error_relf)}")
def testBIWI(model,modelin=args.model,outfile=args.out,feature_transform=args.feat_trans):
if modelin != "":
model.load_state_dict(torch.load(modelin))
model.cuda()
# load 3dmm data
data3dmm = dataloader.SyntheticLoader()
mu_lm = torch.from_numpy(data3dmm.mu_lm).float().cuda()
lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float().cuda()
loader = dataloader.BIWILoader()
seterror_3d = []
seterror_rel3d = []
seterror_relf = []
seterror_2d = []
for sub in range(len(loader)):
batch = loader[sub]
x_cam_gt = batch['x_cam_gt'].cuda()
x_w_gt = batch['x_w_gt'].cuda()
f_gt = batch['f_gt'].cuda()
x_img = batch['x_img'].cuda()
x_img_gt = batch['x_img_gt'].cuda()
M = x_img_gt.shape[0]
one = torch.ones(M,1,68).cuda()
x_img_one = torch.cat([x_img,one],dim=1)
# run the model
out, trans, transfeat = model(x_img_one)
alphas = out[:,:199].mean(0)
f = torch.relu(out[:,199]).mean()
K = torch.zeros((3,3)).float().cuda()
K[0,0] = f;
K[1,1] = f;
K[2,2] = 1;
K[0,2] = 320;
K[1,2] = 240;
# apply 3DMM model from predicted parameters
alpha_matrix = torch.diag(alphas)
shape_cov = torch.mm(lm_eigenvec,alpha_matrix)
s = shape_cov.sum(1).view(68,3)
shape = (mu_lm + s)
shae[:,2] = shape[:,2]*-1
# run epnp algorithm
# get control points
c_w = util.getControlPoints(shape)
# solve alphas
alphas = util.solveAlphas(shape,c_w)
# setup M
px = 320;
py = 240;
Matrix = util.setupM(alphas,x_img.permute(0,2,1),px,py,f)
# get eigenvectors of M for each view
u,d,v = torch.svd(Matrix)
#solve N=1
c_c_n1 = v[:,:,-1].reshape((M,4,3)).permute(0,2,1)
_ , x_c_n1, _ = util.scaleControlPoints(c_c_n1,c_w[:3,:],alphas,shape)
Rn1,Tn1 = util.getExtrinsics(x_c_n1,shape)
reproj_error2_n1 = util.getReprojError2(x_img,shape,Rn1,Tn1,K)
reproj_error3_n1 = util.getReprojError3(x_cam_gt,shape,Rn1,Tn1)
rel_error_n1 = util.getRelReprojError3(x_cam_gt,shape,Rn1,Tn1)
# solve N=2
# get distance contraints
d12,d13,d14,d23,d24,d34 = util.getDistances(c_w)
distances = torch.stack([d12,d13,d14,d23,d24,d34])**2
beta_n2 = util.getBetaN2(v[:,:,-2:],distances)
c_c_n2 = util.getControlPointsN2(v[:,:,-2:],beta_n2)
_,x_c_n2,_ = util.scaleControlPoints(c_c_n2,c_w[:3,:],alphas,shape)
Rn2,Tn2 = util.getExtrinsics(x_c_n2,shape)
reproj_error2_n2 = util.getReprojError2(x_img,shape,Rn2,Tn2,K)
reproj_error3_n2 = util.getReprojError3(x_cam_gt,shape,Rn2,Tn2)
rel_error_n2 = util.getRelReprojError3(x_cam_gt,shape,Rn1,Tn1)
mask = reproj_error2_n1 < reproj_error2_n2
reproj_errors = torch.cat((reproj_error2_n1[mask],reproj_error2_n2[~mask]))
rmse_errors = torch.cat((reproj_error3_n1[mask],reproj_error3_n2[~mask]))
rel_errors = torch.cat((rel_error_n2[~mask],rel_error_n1[mask]))
# errors
reproj_error = torch.mean(reproj_errors)
reconstruction_error = torch.mean(rmse_errors)
rel_error = torch.mean(rel_errors)
f_error = torch.abs(f_gt - f) / f_gt
seterror_2d.append(reproj_error.cpu().data.item())
seterror_3d.append(reconstruction_error.cpu().data.item())
seterror_rel3d.append(rel_error.cpu().data.item())
seterror_relf.append(f_error.cpu().data.item())
print(f"fgt: {f_gt.mean().item():.3f} | f_error_rel: {f_error.item():.4f} | rmse: {reconstruction_error.item():.4f} | rel rmse: {rel_error.item():.4f} | 2d error: {reproj_error.item():.4f}")
#end for
matdata = {}
matdata['seterror_2d'] = np.array(seterror_2d)
matdata['seterror_3d'] = np.array(seterror_3d)
matdata['seterror_rel3d'] = np.array(seterror_rel3d)
matdata['seterror_relf'] = np.array(seterror_relf)
scipy.io.savemat(outfile,matdata)
print(f"MEAN seterror_2d: {np.mean(seterror_2d)}")
print(f"MEAN seterror_3d: {np.mean(seterror_3d)}")
print(f"MEAN seterror_rel3d: {np.mean(seterror_rel3d)}")
print(f"MEAN seterror_relf: {np.mean(seterror_relf)}")
