def test(modelin=args.model,outfile=args.out,optimize=args.opt,ft=args.ft):
# define model, dataloader, 3dmm eigenvectors, optimization method
calib_net = PointNet(n=1,feature_transform=ft)
sfm_net = PointNet(n=199,feature_transform=ft)
if modelin != "":
calib_path = os.path.join('model','calib_' + modelin)
sfm_path = os.path.join('model','sfm_' + modelin)
calib_net.load_state_dict(torch.load(calib_path))
sfm_net.load_state_dict(torch.load(sfm_path))
calib_net.eval()
sfm_net.eval()
# mean shape and eigenvectors for 3dmm
M = 100
data3dmm = dataloader.SyntheticLoader()
mu_lm = torch.from_numpy(data3dmm.mu_lm).float().detach()
mu_lm[:,2] = mu_lm[:,2]*-1
lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float().detach()
sigma = torch.from_numpy(data3dmm.sigma).float().detach()
sigma = torch.diag(sigma.squeeze())
lm_eigenvec = torch.mm(lm_eigenvec, sigma)
# sample from f testing set
allerror_2d = []
allerror_3d = []
allerror_rel3d = []
allerror_relf = []
all_f = []
all_fpred = []
all_depth = []
out_shape = []
out_f = []
seterror_3d = []
seterror_rel3d = []
seterror_relf = []
seterror_2d = []
f_vals = [i*100 for i in range(4,15)]
# set random seed for reproducibility of test set
np.random.seed(0)
torch.manual_seed(0)
for f_test in f_vals:
# create dataloader
loader = dataloader.TestLoader(f_test)
f_pred = []
shape_pred = []
error_2d = []
error_3d = []
error_rel3d = []
error_relf = []
M = 100;
N = 68;
batch_size = 1;
for j,data in enumerate(loader):
if j >= 10: break
# load the data
x_cam_gt = data['x_cam_gt']
shape_gt = data['x_w_gt']
fgt = data['f_gt']
x_img = data['x_img']
x_img_gt = data['x_img_gt']
depth = torch.norm(x_cam_gt.mean(2),dim=1)
all_depth.append(depth.numpy())
all_f.append(fgt.numpy()[0])
ptsI = x_img.reshape((M,N,2)).permute(0,2,1)
x = x_img.unsqueeze(0).permute(0,2,1)
# run the model
f = calib_net(x) + 300
betas = sfm_net(x)
betas = betas.squeeze(0).unsqueeze(-1)
shape = mu_lm + torch.mm(lm_eigenvec,betas).squeeze().view(N,3)
shape = shape - shape.mean(0).unsqueeze(0)
# get motion measurement guess
K = torch.zeros((3,3)).float()
K[0,0] = f
K[1,1] = f
K[2,2] = 1
km,c_w,scaled_betas,alphas = util.EPnP(ptsI,shape,K)
_, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI)
error_time = util.getTimeConsistency(shape,R,T)
if error_time > 10:
mode='walk'
else:
mode='still'
# apply dual optimization
if optimize:
calib_net.load_state_dict(torch.load(calib_path))
sfm_net.load_state_dict(torch.load(sfm_path))
shape,K,R,T = dualoptimization(x,calib_net,sfm_net,shape_gt=shape_gt,fgt=fgt,mode=mode)
f = K[0,0].detach()
else:
K = torch.zeros(3,3).float()
K[0,0] = f
K[1,1] = f
K[2,2] = 1
km,c_w,scaled_betas,alphas = util.EPnP(ptsI,shape,K)
Xc, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI)
# get errors
reproj_errors2 = util.getReprojError2(ptsI,shape,R,T,K,show=False)
reproj_errors3 = torch.norm(shape_gt - shape,dim=1).mean()
rel_errors = util.getRelReprojError3(x_cam_gt,shape,R,T)
reproj_error = reproj_errors2.mean()
reconstruction_error = reproj_errors3.mean()
rel_error = rel_errors.mean()
f_error = torch.abs(fgt - f) / fgt
# save final prediction
f_pred.append(f.detach().cpu().item())
shape_pred.append(shape.detach().cpu().numpy())
all_fpred.append(f.detach().data.numpy())
allerror_3d.append(reproj_error.data.numpy())
allerror_2d.append(reconstruction_error.data.numpy())
allerror_rel3d.append(rel_error.data.numpy())
error_2d.append(reproj_error.cpu().data.item())
error_3d.append(reconstruction_error.cpu().data.item())
error_rel3d.append(rel_error.cpu().data.item())
error_relf.append(f_error.cpu().data.item())
print(f"f/sequence: {f_test}/{j} | f/fgt: {f.item():.3f}/{fgt.item():.3f} | f_error_rel: {f_error.item():.4f} | rmse: {reconstruction_error.item():.4f} | rel rmse: {rel_error.item():.4f} | 2d error: {reproj_error.item():.4f}")
avg_2d = np.mean(error_2d)
avg_rel3d = np.mean(error_rel3d)
avg_3d = np.mean(error_3d)
avg_relf = np.mean(error_relf)
seterror_2d.append(avg_2d)
seterror_3d.append(avg_3d)
seterror_rel3d.append(avg_rel3d)
seterror_relf.append(avg_relf)
out_f.append(np.stack(f_pred))
out_shape.append(np.concatenate(shape_pred,axis=0))
print(f"f_error_rel: {avg_relf:.4f} | rel rmse: {avg_rel3d:.4f} | 2d error: {avg_2d:.4f} | rmse: {avg_3d:.4f} |")
# save output
out_shape = np.stack(out_shape)
out_f = np.stack(out_f)
all_f = np.stack(all_f).flatten()
all_fpred = np.stack(all_fpred).flatten()
all_d = np.stack(all_depth).flatten()
allerror_2d = np.stack(allerror_2d).flatten()
allerror_rel3d = np.stack(allerror_rel3d).flatten()
matdata = {}
matdata['fvals'] = np.array(f_vals)
matdata['all_f'] = np.array(all_f)
matdata['all_fpred'] = np.array(all_fpred)
matdata['all_d'] = np.array(all_depth)
matdata['error_2d'] = allerror_2d
matdata['error_3d'] = allerror_3d
matdata['error_rel3d'] = allerror_rel3d
matdata['seterror_2d'] = np.array(seterror_2d)
matdata['seterror_3d'] = np.array(seterror_3d)
matdata['seterror_rel3d'] = np.array(seterror_rel3d)
matdata['seterror_relf'] = np.array(seterror_relf)
matdata['shape'] = np.stack(out_shape)
matdata['f'] = np.stack(out_f)
scipy.io.savemat(outfile,matdata)
print(f"MEAN seterror_2d: {np.mean(seterror_2d)}")
print(f"MEAN seterror_3d: {np.mean(seterror_3d)}")
print(f"MEAN seterror_rel3d: {np.mean(seterror_rel3d)}")
print(f"MEAN seterror_relf: {np.mean(seterror_relf)}")
return np.mean(seterror_relf)
def testReal(modelin=args.model,outfile=args.out,optimize=args.opt,db=args.db):
# define model, dataloader, 3dmm eigenvectors, optimization method
calib_net = PointNet(n=1)
sfm_net = PointNet(n=199)
if modelin != "":
calib_path = os.path.join('model','calib_' + modelin)
sfm_path = os.path.join('model','sfm_' + modelin)
calib_net.load_state_dict(torch.load(calib_path))
sfm_net.load_state_dict(torch.load(sfm_path))
calib_net.eval()
sfm_net.eval()
# mean shape and eigenvectors for 3dmm
data3dmm = dataloader.SyntheticLoader()
mu_lm = torch.from_numpy(data3dmm.mu_lm).float().detach()
mu_lm[:,2] = mu_lm[:,2]*-1
lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float().detach()
sigma = torch.from_numpy(data3dmm.sigma).float().detach()
sigma = torch.diag(sigma.squeeze())
lm_eigenvec = torch.mm(lm_eigenvec, sigma)
# define loader
loader = getLoader(db)
f_pred = []
shape_pred = []
error_2d = []
error_relf = []
error_rel3d = []
for sub in range(len(loader)):
batch = loader[sub]
x_cam_gt = batch['x_cam_gt']
fgt = batch['f_gt']
x_img = batch['x_img']
x_img_gt = batch['x_img_gt']
M = x_img_gt.shape[0]
N = x_img_gt.shape[-1]
ptsI = x_img.reshape((M,N,2)).permute(0,2,1)
x = x_img.unsqueeze(0).permute(0,2,1)
# run the model
f = calib_net(x) + 300
betas = sfm_net(x)
betas = betas.squeeze(0).unsqueeze(-1)
shape = mu_lm + torch.mm(lm_eigenvec,betas).squeeze().view(N,3)
shape = shape - shape.mean(0).unsqueeze(0)
# get motion measurement guess
K = torch.zeros((3,3)).float()
K[0,0] = f
K[1,1] = f
K[2,2] = 1
km,c_w,scaled_betas,alphas = util.EPnP(ptsI,shape,K)
_, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI)
error_time = util.getTimeConsistency(shape,R,T)
if error_time > 20:
mode='walk'
else:
mode='still'
# adjust number of landmarks
M = x_img_gt.shape[0]
N = x_img_gt.shape[-1]
# additional optimization on initial solution
if optimize:
calib_net.load_state_dict(torch.load(calib_path))
sfm_net.load_state_dict(torch.load(sfm_path))
if db == 'biwi':
shape_gt = batch['x_w_gt']
shape,K,R,T = dualoptimization(x,calib_net,sfm_net,shape_gt=shape_gt,fgt=fgt,M=M,N=N,mode=mode,db='biwi')
else:
shape,K,R,T = dualoptimization(x,calib_net,sfm_net,fgt=fgt,M=M,N=N,mode=mode)
f = K[0,0].detach()
else:
K = torch.zeros(3,3).float()
K[0,0] = f
K[1,1] = f
K[2,2] = 1
km,c_w,scaled_betas,alphas = util.EPnP(ptsI,shape,K)
Xc, R, T, mask = util.optimizeGN(km,c_w,scaled_betas,alphas,shape,ptsI)
# get errors
reproj_errors2 = util.getReprojError2(ptsI,shape,R,T,K)
rel_errors = util.getRelReprojError3(x_cam_gt,shape,R,T)
reproj_error = reproj_errors2.mean()
rel_error = rel_errors.mean()
f_error = torch.abs(fgt - f) / fgt
# save final prediction
f_pred.append(f.detach().cpu().item())
shape_pred.append(shape.detach().cpu().numpy())
error_2d.append(reproj_error.cpu().data.item())
error_rel3d.append(rel_error.cpu().data.item())
error_relf.append(f_error.cpu().data.item())
print(f" f/fgt: {f.item():.3f}/{fgt.item():.3f} | f_error_rel: {f_error.item():.4f} | rel rmse: {rel_error.item():.4f} | 2d error: {reproj_error.item():.4f}")
#end for
# prepare output file
out_shape = np.stack(shape_pred)
out_f = np.stack(f_pred)
matdata = {}
matdata['shape'] = np.stack(out_shape)
matdata['f'] = np.stack(out_f)
matdata['error_2d'] = np.array(error_2d)
matdata['error_rel3d'] = np.array(error_rel3d)
matdata['error_relf'] = np.array(error_relf)
scipy.io.savemat(outfile,matdata)
print(f"MEAN seterror_2d: {np.mean(error_2d)}")
print(f"MEAN seterror_rel3d: {np.mean(error_rel3d)}")
print(f"MEAN seterror_relf: {np.mean(error_relf)}")
def train(args, io):
file_list = [
'preprocess1_HHESTIA_ttbar', 'preprocess2_HHESTIA_RadionToZZ',
'preprocess_HHESTIA_QCD1800To2400', 'preprocess0_HHESTIA_HH_4B',
'preprocess2_HHESTIA_ZprimeWW'
]
train_grp_list = ['train_group' for i in range(len(file_list))]
test_grp_list = ['test_group' for i in range(len(file_list))]
data_dir = '/afs/crc.nd.edu/user/a/adas/DGCNN'
pcl_train_dataset = Jet_PointCloudDataSet(file_list, train_grp_list,
data_dir)
pcl_test_dataset = Jet_PointCloudDataSet(file_list, test_grp_list,
data_dir)
train_loader = DataLoader(pcl_train_dataset,
num_workers=8,
batch_size=args.batch_size,
shuffle=True,
drop_last=True)
test_loader = DataLoader(pcl_test_dataset,
num_workers=8,
batch_size=args.test_batch_size,
shuffle=True,
drop_last=False)
device = torch.device("cuda" if args.cuda else "cpu")
if args.model == 'pointnet':
model = PointNet(args).to(device)
elif args.model == 'dgcnn':
model = FrameImageModel(args).to(device)
elif args.model == 'jetClassifier':
model = JetClassifier(args).to(device)
else:
raise Exception("Not implemented")
#print(str(model))
model = model.double()
model = nn.DataParallel(model)
torch.save(model.state_dict(), 'model.pt')
print("Let's use", torch.cuda.device_count(), "GPUs!")
if args.use_sgd:
print("Use SGD")
opt = optim.SGD(model.parameters(),
lr=args.lr * 100,
momentum=args.momentum,
weight_decay=1e-4)
else:
print("Use Adam")
opt = optim.Adam(model.parameters(), lr=args.lr, weight_decay=1e-4)
scheduler = CosineAnnealingLR(opt, args.epochs, eta_min=args.lr)
criterion = cal_loss
best_test_acc = 0
record_file = open("note.txt", "w")
for epoch in range(args.epochs):
scheduler.step()
####################
# Train
####################
print("########## training on epoch number ------>> ", epoch)
train_loss = 0.0
count = 0.0
model.train()
train_pred = []
train_true = []
batch_number = 1
for data, label in train_loader:
data, label = data.to(device), label.to(device).squeeze()
data = data.permute(0, 2, 1).double()
batch_size = data.size()[0]
opt.zero_grad()
#print(data.shape)
logits = model(data)
loss = criterion(logits, label)
batch_loss = loss.detach().cpu().numpy()
print("### batch number ", batch_number, " ", loss)
binarized_label = label_binarize(label.cpu().numpy(),
classes=[0, 1, 2, 3, 4])
batch_number = batch_number + 1
loss.backward()
opt.step()
preds = logits.max(dim=1)[1]
batch_label = label.detach().cpu().numpy()
batch_preds = preds.detach().cpu().numpy()
batch_acc = metrics.accuracy_score(batch_label, batch_preds)
balanced_batch_acc = metrics.balanced_accuracy_score(
batch_label, batch_preds)
print("### batch accuracy scores ", batch_acc, " ",
balanced_batch_acc)
record_file.write(
str(epoch) + " " + str(batch_number) + " " + str(batch_loss) +
" " + str(batch_acc) + " " + str(balanced_batch_acc))
count += batch_size
train_loss += loss.item() * batch_size
train_true.append(label.cpu().numpy())
train_pred.append(preds.detach().cpu().numpy())
train_true = np.concatenate(train_true)
train_pred = np.concatenate(train_pred)
accuracy_score = metrics.accuracy_score(train_true, train_pred)
balanced_accuracy_score = metrics.balanced_accuracy_score(
train_true, train_pred)
print(accuracy_score, balanced_accuracy_score)
outstr = 'Train %d, loss: %.6f, train acc: %.6f, train avg acc: %.6f' % (
epoch, train_loss * 1.0 / count, accuracy_score,
balanced_accuracy_score)
io.cprint(outstr)
####################
# Test
####################
test_loss = 0.0
count = 0.0
model.eval()
test_pred = []
test_true = []
for data, label in test_loader:
#print(data.shape)
data, label = data.to(device), label.to(device).squeeze()
data = data.permute(0, 2, 1)
batch_size = data.size()[0]
logits = model(data)
loss = criterion(logits, label)
preds = logits.max(dim=1)[1]
batch_label = label.detach().cpu().numpy()
batch_preds = preds.detach().cpu().numpy()
batch_acc = metrics.accuracy_score(batch_label, batch_preds)
balanced_batch_acc = metrics.balanced_accuracy_score(
batch_label, batch_preds)
print("### test batch accuracy scores ", batch_acc, " ",
balanced_batch_acc)
count += batch_size
test_loss += loss.item() * batch_size
test_true.append(label.cpu().numpy())
test_pred.append(preds.detach().cpu().numpy())
test_true = np.concatenate(test_true)
test_pred = np.concatenate(test_pred)
test_acc = metrics.accuracy_score(test_true, test_pred)
avg_per_class_acc = metrics.balanced_accuracy_score(
test_true, test_pred)
outstr = 'Test %d, loss: %.6f, test acc: %.6f, test avg acc: %.6f' % (
epoch, test_loss * 1.0 / count, test_acc, avg_per_class_acc)
io.cprint(outstr)
if test_acc >= best_test_acc:
best_test_acc = test_acc
torch.save(model.state_dict(),
'checkpoints/%s/models/model.t7' % args.exp_name)
record_file.close()
def test_sfm(modelin=args.model, outfile=args.out, optimize=args.opt):
# define model, dataloader, 3dmm
sfm_net = PointNet(n=199)
calib_net = PointNet(n=1)
if modelin != "":
calib_path = os.path.join('model', 'calib_' + modelin)
sfm_path = os.path.join('model', 'sfm_' + modelin)
calib_net.load_state_dict(torch.load(calib_path))
sfm_net.load_state_dict(torch.load(sfm_path))
calib_net.eval()
sfm_net.eval()
# mean shape and eigenvectors for 3dmm
M = 100
data3dmm = dataloader.SyntheticLoader()
mu_lm = torch.from_numpy(data3dmm.mu_lm).float().detach()
mu_lm[:, 2] = mu_lm[:, 2] * -1
lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float().detach()
sigma = torch.from_numpy(data3dmm.sigma).float().detach()
sigma = torch.diag(sigma.squeeze())
lm_eigenvec = torch.mm(lm_eigenvec, sigma)
# sample from f testing set
allerror_2d = []
allerror_3d = []
allerror_rel3d = []
allerror_relf = []
all_f = []
all_fpred = []
all_depth = []
out_shape = []
out_f = []
seterror_3d = []
seterror_rel3d = []
seterror_relf = []
seterror_2d = []
f_vals = [i * 100 for i in range(4, 15)]
for f_test in f_vals:
# create dataloader
#f_test = 1000
loader = dataloader.TestLoader(f_test, addnoise=True)
f_pred = []
shape_pred = []
error_2d = []
error_3d = []
error_rel3d = []
error_relf = []
loader.M = 100
loader.N = 68
batch_size = 1
M = loader.M
N = loader.N
#training_pred = np.zeros((5,M,68,3))
#training_gt = np.zeros((5,M,68,3))
for j, data in enumerate(loader):
if j == 1: break
# load the data
x_cam_gt = data['x_cam_gt']
shape_gt = data['x_w_gt']
fgt = data['f_gt']
x_img = data['x_img']
x_img_gt = data['x_img_gt']
T_gt = data['T_gt']
all_depth.append(np.mean(T_gt[:, 2]))
all_f.append(fgt.numpy()[0])
ptsI = x_img.reshape((M, N, 2)).permute(0, 2, 1)
x = x_img.unsqueeze(0).permute(0, 2, 1)
# test sfm calibration
#calib_net.load_state_dict(torch.load(calib_path))
opt2 = torch.optim.Adam(sfm_net.parameters(), lr=1)
sfm_net.eval()
trainfc(sfm_net)
f = fgt
#v = pptk.viewer(mu_lm.cpu().numpy())
l_init = 100000
for iter in itertools.count():
opt2.zero_grad()
# shape prediction
betas = sfm_net(ptsI).mean(0).unsqueeze(1)
shape = mu_lm + torch.mm(lm_eigenvec, betas).squeeze().view(
N, 3)
shape = shape - shape.mean(0).unsqueeze(0)
rmse = torch.norm(shape_gt - shape, dim=1).mean().detach()
K = torch.zeros((3, 3)).float()
K[0, 0] = f
K[1, 1] = f
K[2, 2] = 1
km, c_w, scaled_betas, alphas = util.EPnP(ptsI, shape, K)
Xc, R, T, mask = util.optimizeGN(km, c_w, scaled_betas, alphas,
shape, ptsI)
error2d = util.getReprojError2(ptsI,
shape,
R,
T,
K,
show=False,
loss='l2')
error_time = util.getTimeConsistency(shape, R, T)
loss = error2d.mean() + 0.01 * error_time
loss.backward()
opt2.step()
print(
f"iter: {iter} | error: {loss.item():.3f} | error2d: {error2d.mean().item():.3f} | rmse: {rmse.item():.3f} "
)
if iter >= 10 and l_init - loss < 0.0001: break
print(l_init - loss)
l_init = loss
#training_pred[j,iter,:,:] = shape.detach().cpu()
#training_gt[j,iter,:,:] = shape_gt.detach().cpu()
all_fpred.append(f.detach().numpy()[0])
# get errors
reproj_errors2 = util.getReprojError2(ptsI,
shape,
R,
T,
K,
show=False)
reproj_errors3 = torch.norm(shape_gt - shape, dim=1).mean()
rel_errors = util.getRelReprojError3(x_cam_gt, shape, R, T)
reproj_error = reproj_errors2.mean()
reconstruction_error = reproj_errors3.mean()
rel_error = rel_errors.mean()
f_error = torch.abs(fgt - f) / fgt
# save final prediction
f_pred.append(f.detach().cpu().item())
shape_pred.append(shape.detach().cpu().numpy())
allerror_3d.append(reproj_error.data.numpy())
allerror_2d.append(reconstruction_error.data.numpy())
allerror_rel3d.append(rel_error.data.numpy())
error_2d.append(reproj_error.cpu().data.item())
error_3d.append(reconstruction_error.cpu().data.item())
error_rel3d.append(rel_error.cpu().data.item())
error_relf.append(f_error.cpu().data.item())
print(
f"f/sequence: {f_test}/{j} | f/fgt: {f[0].item():.3f}/{fgt.item():.3f} | f_error_rel: {f_error.item():.4f} | rmse: {reconstruction_error.item():.4f} | rel rmse: {rel_error.item():.4f} | 2d error: {reproj_error.item():.4f}"
)
avg_2d = np.mean(error_2d)
avg_rel3d = np.mean(error_rel3d)
avg_3d = np.mean(error_3d)
avg_relf = np.mean(error_relf)
seterror_2d.append(avg_2d)
seterror_3d.append(avg_3d)
seterror_rel3d.append(avg_rel3d)
seterror_relf.append(avg_relf)
out_f.append(np.stack(f_pred))
out_shape.append(np.stack(shape_pred, axis=0))
print(
f"f_error_rel: {avg_relf:.4f} | rel rmse: {avg_rel3d:.4f} | 2d error: {reproj_error.item():.4f} | rmse: {avg_3d:.4f} |"
)
out_f = np.stack(out_f)
all_f = np.stack(all_f).flatten()
all_fpred = np.stack(all_fpred).flatten()
all_d = np.stack(all_depth).flatten()
allerror_2d = np.stack(allerror_2d).flatten()
allerror_3d = np.stack(allerror_3d).flatten()
allerror_rel3d = np.stack(allerror_rel3d).flatten()
matdata = {}
#matdata['training_pred'] = training_pred
#matdata['training_gt'] = training_gt
matdata['fvals'] = np.array(f_vals)
matdata['all_f'] = np.array(all_f)
matdata['all_fpred'] = np.array(all_fpred)
matdata['all_d'] = np.array(all_depth)
matdata['error_2d'] = allerror_2d
matdata['error_3d'] = allerror_3d
matdata['error_rel3d'] = allerror_rel3d
matdata['seterror_2d'] = np.array(seterror_2d)
matdata['seterror_3d'] = np.array(seterror_3d)
matdata['seterror_rel3d'] = np.array(seterror_rel3d)
matdata['seterror_relf'] = np.array(seterror_relf)
matdata['f'] = np.stack(out_f)
scipy.io.savemat(outfile, matdata)
print(f"MEAN seterror_2d: {np.mean(seterror_2d)}")
print(f"MEAN seterror_3d: {np.mean(seterror_3d)}")
print(f"MEAN seterror_rel3d: {np.mean(seterror_rel3d)}")
print(f"MEAN seterror_relf: {np.mean(seterror_relf)}")
def test(modelin=args.model, outfile=args.out, optimize=args.opt):
# define model, dataloader, 3dmm eigenvectors, optimization method
calib_net = PointNet(n=1)
sfm_net = PointNet(n=199)
if modelin != "":
calib_path = os.path.join('model', 'calib_' + modelin)
sfm_path = os.path.join('model', 'sfm_' + modelin)
calib_net.load_state_dict(torch.load(calib_path))
sfm_net.load_state_dict(torch.load(sfm_path))
calib_net.eval()
sfm_net.eval()
68 // 10
Mvals = [i for i in range(1, 100)]
Nvals = [i for i in range(3, 68)]
f_vals = [i * 200 for i in range(2, 7)]
fpred_mean = np.zeros((100, 65, 5, 5))
fpred_med = np.zeros((100, 65, 5, 5))
fpred = np.zeros((100, 65, 5, 5))
factual = np.zeros((100, 65, 5, 5))
depth_error = np.zeros((100, 65, 5, 5))
for i, viewcount in enumerate(Mvals):
for j, ptcount in enumerate(Nvals):
for l, ftest in enumerate(f_vals):
data3dmm = dataloader.MNLoader(M=viewcount,
N=ptcount,
f=ftest,
seed=0)
M = data3dmm.M
N = data3dmm.N
# mean shape and eigenvectors for 3dmm
mu_s = torch.from_numpy(data3dmm.mu_s).float().detach()
mu_s[:, 2] = mu_s[:, 2] * -1
lm_eigenvec = torch.from_numpy(
data3dmm.lm_eigenvec).float().detach()
sigma = torch.from_numpy(data3dmm.sigma).float().detach()
sigma = torch.diag(sigma.squeeze())
lm_eigenvec = torch.mm(lm_eigenvec, sigma)
mu_s = torch.from_numpy(data3dmm.mu_s).float().detach()
mu_s[:, 2] = mu_s[:, 2] * -1
lm_eigenvec = torch.from_numpy(
data3dmm.lm_eigenvec).float().detach()
sigma = torch.from_numpy(data3dmm.sigma).float().detach()
sigma = torch.diag(sigma.squeeze())
lm_eigenvec = torch.mm(lm_eigenvec, sigma)
for k in range(5):
data = data3dmm[k]
# load the data
x_cam_gt = data['x_cam_gt']
shape_gt = data['x_w_gt']
fgt = data['f_gt']
x_img = data['x_img']
x_img_gt = data['x_img_gt']
ptsI = x_img.reshape((M, N, 2)).permute(0, 2, 1)
x = x_img.unsqueeze(0).permute(0, 2, 1)
# run the model
f = torch.squeeze(calib_net(ptsI) + 300)
betas = sfm_net(ptsI)
betas = betas.unsqueeze(-1)
eigenvec = torch.stack(M * [lm_eigenvec])
shape = torch.stack(M * [mu_s]) + torch.bmm(
eigenvec, betas).squeeze().view(M, N, 3)
shape = shape - shape.mean(1).unsqueeze(1)
shape = shape.mean(0)
# get motion measurement guess
K = torch.zeros((M, 3, 3)).float()
K[:, 0, 0] = f
K[:, 1, 1] = f
K[:, 2, 2] = 1
km, c_w, scaled_betas, alphas = util.EPnP_single(
ptsI, shape, K)
_, R, T, mask = util.optimizeGN(km, c_w, scaled_betas,
alphas, shape, ptsI)
error_time = util.getTimeConsistency(shape, R, T)
if error_time > 20:
mode = 'walk'
else:
mode = 'still'
# apply dual optimization
if optimize:
calib_net.load_state_dict(torch.load(calib_path))
sfm_net.load_state_dict(torch.load(sfm_path))
shape, K, R, T = dualoptimization(x,
calib_net,
sfm_net,
lm_eigenvec,
betas,
mu_s,
shape_gt=shape_gt,
fgt=fgt,
M=M,
N=N,
mode=mode)
f = K[0, 0].detach()
# get motion measurement guess
fmu = f.mean()
fmed = f.flatten().median()
K = torch.zeros(M, 3, 3).float()
K[:, 0, 0] = fmu
K[:, 1, 1] = fmu
K[:, 2, 2] = 1
K, _ = torch.median(K, dim=0)
km, c_w, scaled_betas, alphas = util.EPnP(ptsI, shape, K)
Xc, R, T, mask = util.optimizeGN(km, c_w, scaled_betas,
alphas, shape, ptsI)
# get errors
rel_errors = util.getRelReprojError3(x_cam_gt, shape, R, T)
fpred_mean[i, j, l, k] = fmu.detach().cpu().item()
fpred_med[i, j, l, k] = fmed.detach().cpu().item()
factual[i, j, l, k] = fgt.detach().cpu().item()
depth_error[i, j, l, k] = rel_errors.cpu().mean().item()
print(
f"M: {viewcount} | N: {ptcount} | f/fgt: {fpred_mean[i,j,l,k]:.2f}/{factual[i,j,l,k]}"
)
ferror_mu = np.mean(
np.abs(fpred_mean[i, j, l] - factual[i, j, l]) /
factual[i, j, l])
ferror_med = np.mean(
np.abs(fpred_med[i, j, l] - factual[i, j, l]) /
factual[i, j, l])
derror = np.mean(depth_error[i, j, l])
f = np.mean(fpred_mean[i, j, l])
print(
f"M: {viewcount} | N: {ptcount} | fgt: {ftest:.2f} | ferror_mu: {ferror_mu:.2f} | ferror_med: {ferror_med:.2f} | derror: {derror:.2f}"
)
matdata = {}
matdata['fpred_mu'] = fpred_mean
matdata['fpred_med'] = fpred_med
matdata['fgt'] = factual
matdata['derror'] = depth_error
scipy.io.savemat(outfile, matdata)
print(f"saved output to {outfile}")
def testReal(modelin=args.model,
outfile=args.out,
optimize=args.opt,
db=args.db):
# define model, dataloader, 3dmm eigenvectors, optimization method
calib_net = PointNet(n=1)
sfm_net = PointNet(n=199)
if modelin != "":
calib_path = os.path.join('model', 'calib_' + modelin)
sfm_path = os.path.join('model', 'sfm_' + modelin)
calib_net.load_state_dict(torch.load(calib_path))
sfm_net.load_state_dict(torch.load(sfm_path))
calib_net.eval()
sfm_net.eval()
# mean shape and eigenvectors for 3dmm
data3dmm = dataloader.SyntheticLoader()
mu_lm = torch.from_numpy(data3dmm.mu_lm).float().detach()
mu_lm[:, 2] = mu_lm[:, 2] * -1
lm_eigenvec = torch.from_numpy(data3dmm.lm_eigenvec).float().detach()
sigma = torch.from_numpy(data3dmm.sigma).float().detach()
sigma = torch.diag(sigma.squeeze())
lm_eigenvec = torch.mm(lm_eigenvec, sigma)
# define loader
loader = getLoader(db)
out_fpred = []
out_fgt = []
out_dpred = []
out_dgt = []
shape_pred = []
error_2d = []
error_relf = []
error_rel3d = []
for sub in range(len(loader)):
batch = loader[sub]
x_cam_gt = batch['x_cam_gt']
fgt = batch['f_gt']
x_img = batch['x_img']
x_img_gt = batch['x_img_gt']
M = x_img_gt.shape[0]
N = x_img_gt.shape[-1]
ptsI = x_img.reshape((M, N, 2)).permute(0, 2, 1)
x = x_img.unsqueeze(0).permute(0, 2, 1)
# run the model
f = torch.squeeze(calib_net(ptsI) + 300)
betas = sfm_net(ptsI)
betas = betas.unsqueeze(-1)
eigenvec = torch.stack(M * [lm_eigenvec])
shape = torch.stack(M * [mu_lm]) + torch.bmm(
eigenvec, betas).squeeze().view(M, N, 3)
shape = shape - shape.mean(1).unsqueeze(1)
shape = shape.mean(0)
# get motion measurement guess
K = torch.zeros((M, 3, 3)).float()
K[:, 0, 0] = f
K[:, 1, 1] = f
K[:, 2, 2] = 1
km, c_w, scaled_betas, alphas = util.EPnP_single(ptsI, shape, K)
_, R, T, mask = util.optimizeGN(km, c_w, scaled_betas, alphas, shape,
ptsI)
error_time = util.getTimeConsistency(shape, R, T)
if error_time > 20:
mode = 'walk'
else:
mode = 'still'
print(mode, error_time)
# additional optimization on initial solution
shape_gt = batch['x_w_gt'] if db == 'biwi' else None
if optimize:
calib_net.load_state_dict(torch.load(calib_path))
sfm_net.load_state_dict(torch.load(sfm_path))
print(mode)
if db == 'biwi':
shape, K, R, T, iter = dualoptimization(ptsI,
calib_net,
sfm_net,
shape_gt=shape_gt,
fgt=fgt,
db='biwi',
mode=mode)
else:
shape, K, R, T, iter = dualoptimization(ptsI,
calib_net,
sfm_net,
fgt=fgt,
mode=mode)
f = K[:, 0, 0].detach()
# get pose with single intrinsic
fmu = f.mean()
fmed = f.flatten().median()
K = torch.zeros(M, 3, 3).float()
K[:, 0, 0] = fmu
K[:, 1, 1] = fmu
K[:, 2, 2] = 1
km, c_w, scaled_betas, alphas = util.EPnP_single(ptsI, shape, K)
Xc, R, T, mask = util.optimizeGN(km, c_w, scaled_betas, alphas, shape,
ptsI)
# get errors
#reproj_errors2 = util.getError(ptsI,shape,R,T,K)
reproj_errors2 = util.getReprojError2(ptsI,
shape,
R,
T,
K.mean(0),
show=False,
loss='l2')
rel_errors = util.getRelReprojError3(x_cam_gt, shape, R, T)
d = torch.norm(T, dim=1)
dgt = torch.norm(torch.mean(x_cam_gt, dim=2), dim=1)
reproj_error = reproj_errors2.mean()
rel_error = rel_errors.mean()
f_error = torch.mean(torch.abs(fgt - fmu) / fgt)
# save final prediction
out_fpred.append(f.detach().cpu().numpy())
out_fgt.append(fgt.numpy())
out_dpred.append(d.detach().cpu().numpy())
out_dgt.append(dgt.cpu().numpy())
f_x = torch.mean(fmu.detach()).cpu().item()
shape_pred.append(shape.detach().cpu().numpy())
error_2d.append(reproj_error.cpu().data.item())
error_rel3d.append(rel_error.cpu().data.item())
error_relf.append(f_error.cpu().data.item())
print(
f" f/fgt: {f_x:.3f}/{fgt.item():.3f} | f_error_rel: {f_error.item():.4f} | rel rmse: {rel_error.item():.4f} | 2d error: {reproj_error.item():.4f}"
)
#end for
# prepare output file
out_shape = np.stack(shape_pred)
out_fpred = np.array(out_fpred, dtype=np.object)
out_fgt = np.array(out_fgt, dtype=np.object).T
matdata = {}
matdata['fpred'] = out_fpred
matdata['fgt'] = out_fgt
matdata['dpred'] = out_dpred
matdata['dgt'] = out_dgt
matdata['shape'] = np.stack(out_shape)
matdata['error_2d'] = np.array(error_2d)
matdata['error_rel3d'] = np.array(error_rel3d)
matdata['error_relf'] = np.array(error_relf)
scipy.io.savemat(outfile, matdata)
print(f"MEAN seterror_2d: {np.mean(error_2d)}")
print(f"MEAN seterror_rel3d: {np.mean(error_rel3d)}")
print(f"MEAN seterror_relf: {np.mean(error_relf)}")