def run_e2e_split(e2e, data, location):
e2e.eval()
images = []
gts = []
hg_preds = []
pdm_preds = []
pdm_3d = []
pdm_applied = []
pdm_encoder_preds = []
for batch in data:
image = move2device(batch['image'].float(), location)
gt = move2device(batch['landmarks'].float(), location)
with torch.no_grad():
pdm_pred, hg_pred, pdm_res_3d, pdm_zs, pdm_nr, apply_pdm, pdm_encoder_pred = e2e(image)
images.append(image)
gts.append(gt)
hg_preds.append(hg_pred)
pdm_preds.append(pdm_pred)
pdm_3d.append(pdm_res_3d)
pdm_applied.append(apply_pdm)
if pdm_encoder_pred is not None:
pdm_encoder_preds.append(pdm_encoder_pred)
images = torch.cat(images)
gts = torch.cat(gts)
hg_preds = torch.cat(hg_preds)
pdm_preds = torch.cat(pdm_preds)
pdm_3d = torch.cat(pdm_3d)
pdm_applied = torch.cat(pdm_applied)
eval_hg = evaluate(hg_preds, gts)
eval_pdm = evaluate(pdm_preds, gts)
res = {
"images": images,
"gt": gts,
"hg_pred": hg_preds,
"pdm_pred": pdm_pred,
"pdm_3d": pdm_3d,
"eval_hg": eval_hg,
"eval_pdm": eval_pdm,
"pdm_applied": pdm_applied
}
if len(pdm_encoder_preds) > 0:
pdm_encoder_preds = torch.cat(pdm_encoder_preds)
eval_pdm_encoder = evaluate(pdm_encoder_pred, gts)
res["pdm_encoder_pred"] = pdm_encoder_preds
res["eval_pdm_encoder"] = eval_pdm_encoder
return res
def get_errors(source, split):
with open(source) as f:
data = json.load(f)
preds = []
gts = []
errors49 = []
errors68 = []
for i, sample in enumerate(data[split]["results"]):
pred = []
gt = []
for landmark in sample["coord_and_variance"]:
pred.append([landmark["pred_x"], landmark["pred_y"]])
gt.append([landmark["gt_x"], landmark["gt_y"]])
vx.append(landmark["var_x"])
vy.append(landmark["var_y"])
preds.append(pred)
gts.append(gt)
if split == "menpo":
outline_error, no_outline_error = evaluate_menpo(
torch.tensor([pred]),
menpo_gt[i:i + 1]) # we need 68 LM to compute the error
errors49.append(no_outline_error / 100)
errors68.append(outline_error / 100)
else:
error = evaluate(torch.tensor([pred]), torch.tensor([gt]))
errors49.append(error["without_outline"] / 100)
errors68.append(error["with_outline"] / 100)
print("NO OUTLINE", np.median(errors49))
print("OUTLINE", np.median(errors68))
return errors49, errors68
def run(pdm, hg_results, gpu):
location = 'cpu' if gpu is None else "cuda:%d" % gpu
data = torch.load(pdm, map_location='cpu')
state_dict = data['state_dict']
config = data['config']
make_deterministic(config['random_seed'])
net = ModelTrainer.create_net(config)
net.model.load_state_dict(state_dict)
net.model.eval()
net.to(location)
net.bs *= 256
hg_out = json.load(open(hg_results, "r"))
#avg_dist = torch.tensor(hg_out["train"]["average_lm_distances"], device=location)
easy = [x["coord_and_conf"] for x in hg_out["easy"]["results"]]
easy_gt = torch.tensor([[[y["gt_x"], y["gt_y"]] for y in x] for x in easy],
device=location)
hard = [x["coord_and_conf"] for x in hg_out["hard"]["results"]]
hard_gt = torch.tensor([[[y["gt_x"], y["gt_y"]] for y in x] for x in hard],
device=location)
train = [x["coord_and_conf"] for x in hg_out["train"]["results"]]
#gauss = norm(0.0, stddev)
#easy_hg_pred = torch.tensor([[[y["pred_x"], y["pred_y"], gauss.pdf(y["dist_x"]), gauss.pdf(y["dist_y"])] for y in x] for x in easy], device=location)
#hard_hg_pred = torch.tensor([[[y["pred_x"], y["pred_y"], gauss.pdf(y["dist_x"]), gauss.pdf(y["dist_y"])] for y in x] for x in hard], device=location)
import math
import random
#mp = lambda x: (-5494.5 * x + 1.099)**2
#mp = lambda x: 1/(100000*x**2+1)
mp = lambda x: min(1, max(0, 1 / x - 130))
mp = lambda x: 1 / x
#print(torch.min(avg_dist), torch.max(avg_dist))
#exit()
"""
varx = torch.tensor([[1/y["var_x"] for y in x] for x in easy], device=location)
vary = torch.tensor([[1/y["var_y"] for y in x] for x in easy], device=location)
print("easy", torch.min(varx), torch.max(varx))
print("easy", torch.min(vary), torch.max(vary))
varx = torch.tensor([[1/y["var_x"] for y in x] for x in hard], device=location)
vary = torch.tensor([[1/y["var_y"] for y in x] for x in hard], device=location)
print("hard", torch.min(varx), torch.max(varx))
print("hard", torch.min(vary), torch.max(vary))
varx = torch.tensor([[1/y["var_x"] for y in x] for x in train], device=location)
vary = torch.tensor([[1/y["var_y"] for y in x] for x in train], device=location)
print("train", torch.min(varx), torch.max(varx))
print("train", torch.min(vary), torch.max(vary))
exit()
"""
#easy_hg_pred = torch.tensor([[[y["pred_x"], y["pred_y"], mp(avg_dist[i][0]), mp(avg_dist[i][1])] for i,y in enumerate(x)] for x in easy], device=location)
#hard_hg_pred = torch.tensor([[[y["pred_x"], y["pred_y"], mp(avg_dist[i][0]), mp(avg_dist[i][1])] for i,y in enumerate(x)] for x in hard], device=location)
easy_hg_pred = torch.tensor(
[[[y["pred_x"], y["pred_y"],
mp(y["var_x"]),
mp(y["var_y"])] for i, y in enumerate(x)] for x in easy],
device=location)
hard_hg_pred = torch.tensor(
[[[y["pred_x"], y["pred_y"],
mp(y["var_x"]),
mp(y["var_y"])] for i, y in enumerate(x)] for x in hard],
device=location)
#print(torch.min(easy_hg_pred[:,:,2:]), torch.max(easy_hg_pred[:,:,2:]))
#print(torch.min(hard_hg_pred[:, :, 2:]), torch.max(hard_hg_pred[:, :, 2:]))
#exit()
sample_losses_hg_easy = [
np.mean((easy_hg_pred[i, :, :2].cpu().numpy() -
easy_gt[i].cpu().numpy())**2) for i in range(easy_gt.shape[0])
]
# TODO test() takes pred and conf now separately
zs, nr, *_ = net.test(easy_hg_pred, verbose=True)
l2d_easy, _ = net.forward(zs, nr)
sample_losses_pdm_easy = [
np.mean((l2d_easy[i].detach().cpu().numpy() -
easy_gt[i].detach().cpu().numpy())**2)
for i in range(easy_gt.shape[0])
]
easy_best = Counter()
best_coords_easy = []
worst_coords_easy = []
for i in range(easy_gt.shape[0]):
if sample_losses_pdm_easy[i] <= sample_losses_hg_easy[i]:
easy_best["pdm"] += 1
best_coords_easy.append(
l2d_easy[i].cpu().detach().numpy().tolist())
worst_coords_easy.append(
easy_hg_pred[i, :, :2].cpu().detach().numpy().tolist())
else:
easy_best["hg"] += 1
best_coords_easy.append(
easy_hg_pred[i, :, :2].cpu().detach().numpy().tolist())
worst_coords_easy.append(
l2d_easy[i].cpu().detach().numpy().tolist())
sample_losses_hg_hard = [
np.mean((hard_hg_pred[i, :, :2].cpu().numpy() -
hard_gt[i].cpu().numpy())**2) for i in range(hard_gt.shape[0])
]
# TODO test() takes pred and conf now separately
zs, nr, *_ = net.test(hard_hg_pred, verbose=True)
l2d_hard, _ = net.forward(zs, nr)
sample_losses_pdm_hard = [
np.mean((l2d_hard[i].detach().cpu().numpy() -
hard_gt[i].detach().cpu().numpy())**2)
for i in range(hard_gt.shape[0])
]
hard_best = Counter()
best_coords_hard = []
worst_coords_hard = []
for i in range(hard_gt.shape[0]):
if sample_losses_pdm_hard[i] <= sample_losses_hg_hard[i]:
hard_best["pdm"] += 1
best_coords_hard.append(
l2d_hard[i].cpu().detach().numpy().tolist())
worst_coords_hard.append(
hard_hg_pred[i, :, :2].cpu().detach().numpy().tolist())
else:
hard_best["hg"] += 1
best_coords_hard.append(
hard_hg_pred[i, :, :2].cpu().detach().numpy().tolist())
worst_coords_hard.append(
l2d_hard[i].cpu().detach().numpy().tolist())
hg_easy_eval = evaluate(easy_hg_pred[:, :, :2], easy_gt)
all_pdm_easy_eval = evaluate(l2d_easy, easy_gt)
best_pick_easy = evaluate(
torch.tensor(best_coords_easy, dtype=torch.float32).cpu(),
easy_gt.cpu())
worst_pick_easy = evaluate(
torch.tensor(worst_coords_easy, dtype=torch.float32).cpu(),
easy_gt.cpu())
print("\n---- EASY without outline----")
print("HG \t\t %0.4f" % hg_easy_eval["without_outline"])
print("best pick \t %0.4f" % best_pick_easy["without_outline"])
print("worst pick \t %0.4f" % worst_pick_easy["without_outline"])
print("all PDM \t %0.4f" % all_pdm_easy_eval["without_outline"])
print("\n---- EASY with outline----")
print("HG \t\t %0.4f" % hg_easy_eval["with_outline"])
print("best pick \t %0.4f" % best_pick_easy["with_outline"])
print("worst pick \t %0.4f" % worst_pick_easy["with_outline"])
print("all PDM \t %0.4f" % all_pdm_easy_eval["with_outline"])
print("easy best", easy_best)
hg_hard_eval = evaluate(hard_hg_pred[:, :, :2], hard_gt)
all_pdm_hard_eval = evaluate(l2d_hard, hard_gt)
best_pick_hard = evaluate(
torch.tensor(best_coords_hard, dtype=torch.float32).cpu(),
hard_gt.cpu())
worst_pick_hard = evaluate(
torch.tensor(worst_coords_hard, dtype=torch.float32).cpu(),
hard_gt.cpu())
print("\n---- HARD without outline----")
print("HG \t\t %0.4f" % hg_hard_eval["without_outline"])
print("best pick \t %0.4f" % best_pick_hard["without_outline"])
print("worst pick \t %0.4f" % worst_pick_hard["without_outline"])
print("all PDM \t %0.4f" % all_pdm_hard_eval["without_outline"])
print("\n---- HARD with outline----")
print("HG \t\t %0.4f" % hg_hard_eval["with_outline"])
print("best pick \t %0.4f" % best_pick_hard["with_outline"])
print("worst pick \t %0.4f" % worst_pick_hard["with_outline"])
print("all PDM \t %0.4f" % all_pdm_hard_eval["with_outline"])
print("hard_best", hard_best)
def run(self):
self.config["model_dir"] = self.model_dir
make_deterministic(self.config['random_seed'])
pdm = ModelTrainer.create_net(self.config)
self.to_gpu(pdm)
pdm.verbose = not self.is_gridsearch
pdm.listener = self.receive_pdm_output
dt = h5py.File(self.data, "r")
data_tr = self.to_gpu(
torch.tensor(dt["300W"]["train_y"], dtype=torch.float32))
data_te = self.to_gpu(
torch.tensor(dt["300W"]["test_y"], dtype=torch.float32))
if self.config["add_multipie"]:
tmp = self.to_gpu(
torch.tensor(dt["multipie"]["train_y"], dtype=torch.float32))
data_tr = torch.cat((data_tr, tmp))
#print("train", data_tr.shape)
#print("test", data_te.shape)
#exit()
zs_tr, nr_tr, loss_tr = pdm.train(data=data_tr)
train_reconstructed, _ = pdm.forward(zs_tr, nr_tr)
zs_te, nr_te, loss_te, *_ = pdm.test(data=data_te, confidence=None)
test_reconstructed, _ = pdm.forward(zs_te, nr_te)
target_file = os.path.join(
self.result_dir, "zs_and_nr_%d.json" % self.config["config_id"])
json.dump(
{
"train": {
"zs":
zs_tr.detach().cpu().numpy().tolist(),
"nr":
nr_tr.detach().cpu().numpy().tolist(),
"reconstructed":
train_reconstructed.detach().cpu().numpy().tolist(),
"coords":
data_tr.detach().cpu().numpy().tolist()
},
"test": {
"zs":
zs_te.detach().cpu().numpy().tolist(),
"nr":
nr_te.detach().cpu().numpy().tolist(),
"reconstructed":
test_reconstructed.detach().cpu().numpy().tolist(),
"coords":
data_te.detach().cpu().numpy().tolist()
}
}, open(target_file, "w"))
pdm.save_pdm(
pdm.train_epochs,
os.path.join(self.model_dir,
"final_pdm_%d.torch" % self.config["config_id"]))
# TODO train ENCODERS DIRECTLY HERE
if self.is_gridsearch:
last_train_loss = self.loss_log["train"][-1]
lowest_train_loss = min(self.loss_log["train"])
best_train_epoch = min([
i for i in range(len(self.loss_log["train"]))
if self.loss_log["train"][i] == lowest_train_loss
])
train_error = evaluate(train_reconstructed, data_tr)
test_error = evaluate(test_reconstructed, data_te)
#print(train_error, test_error)
best_epochs = {
"best_%s_epoch" % k: v
for k, v in self.best_epoch.items()
}
best_errors = {
"best_%s" % k: v
for k, v in self.lowest_error.items()
}
return {
**self.config, "last_train_loss": last_train_loss,
"lowest_train_loss": lowest_train_loss,
"best_train_epoch": best_train_epoch,
"metrics_log": self.metrics_log,
**best_epochs,
**best_errors, "train_error_49":
train_error["without_outline"],
"train_error_68": train_error["with_outline"],
"test_error_49": test_error["without_outline"],
"test_error_68": test_error["with_outline"]
}
else:
# evaluate PDM
metrics = pdm.eval_on_alpha_hg()
print(metrics["easy_metrics_last"])
print(metrics["hard_metrics_last"])
gt_lm = batch['landmarks']
img = batch['image'].to(location)
starttime = time.time()
predicted_landmarks, heatmaps, var, unnormal_hms = net(img)
endtime = time.time()
duration_batch = endtime - starttime
durations.append(duration_batch)
#print(duration_batch)
gts.append(gt_lm.detach())
preds.append(predicted_landmarks.detach())
print("mean duration per batch", sum(durations) / len(durations))
print()
print("Menpo with BB size error")
res_menpo = evaluate_menpo(
torch.cat(preds).cpu().float(),
torch.cat(gts).cpu().float())
print("Without outline", res_menpo[1])
print("With outline", res_menpo[0])
print()
res_menpo_iod = evaluate(
torch.cat(preds).cpu().float(),
torch.cat(gts).cpu().float())
print("menpo with IOD error")
print("49", res_menpo_iod["without_outline"])
print("68", res_menpo_iod["with_outline"])
tmp.append((xs[i][k].item(), ys[i][k].item()))
coordpreds.append(tmp)
ibug_pred = coordpreds[0:135]
lfpw_pred = coordpreds[135:359]
helen_pred = coordpreds[359:689]
helen_lfpw_pred = helen_pred + lfpw_pred
gt_data = sio.loadmat(gt_mat_src)["W300_gt"].transpose(2, 1, 0).tolist()
ibug_gt = gt_data[0:135]
lfpw_gt = gt_data[135:359]
helen_gt = gt_data[359:689]
helen_lfpw_gt = helen_gt + lfpw_gt
res_ibug = evaluate(
torch.tensor(ibug_pred).float(),
torch.tensor(ibug_gt).float())
print("ibug", res_ibug)
res_lfpw = evaluate(
torch.tensor(lfpw_pred).float(),
torch.tensor(lfpw_gt).float())
print("lfpw", res_lfpw)
res_helen = evaluate(
torch.tensor(helen_pred).float(),
torch.tensor(helen_gt).float())
print("helen", res_helen)
res_helen_lfpw = evaluate(
torch.tensor(helen_lfpw_pred).float(),
def evaluate_predictions_split(split):
gt = torch.tensor(split["gt"])
pred = torch.tensor(split["pdm_pred"])
res = evaluate(pred, gt)
return res
def eval_on_alpha_hg(self):
self.init_hg_results()
eval_bs = 2048
# do 49LM easy
d = self.alpha_hg_data["49_lm"]["easy"]
gt, hg_pred, hg_conf = d["gt"], d["hg_pred"], d["hg_conf"]
zs, nr, easy_loss, zs_best, nr_best, best_epochs_easy, sample_losses, _, _ = self.test(
hg_pred.to(self.device, copy=True),
hg_conf.to(self.device, copy=True),
verbose=False,
bs=eval_bs)
l2d, _ = self.forward(zs, nr)
easy49_err = evaluate(l2d, gt.to(self.device,
copy=True))["without_outline"]
torch.cuda.empty_cache()
# do 49LM hard
d = self.alpha_hg_data["49_lm"]["hard"]
gt, hg_pred, hg_conf = d["gt"], d["hg_pred"], d["hg_conf"]
zs, nr, easy_loss, zs_best, nr_best, best_epochs_easy, sample_losses, _, _ = self.test(
hg_pred.to(self.device, copy=True),
hg_conf.to(self.device, copy=True),
verbose=False,
bs=eval_bs)
l2d, _ = self.forward(zs, nr)
hard49_err = evaluate(l2d, gt.to(self.device,
copy=True))["without_outline"]
torch.cuda.empty_cache()
if self.is_68_pdm:
# do 68LM easy
d = self.alpha_hg_data["68_lm"]["easy"]
gt, hg_pred, hg_conf = d["gt"], d["hg_pred"], d["hg_conf"]
zs, nr, easy_loss, zs_best, nr_best, best_epochs_easy, sample_losses, _, _ = self.test(
hg_pred.to(self.device, copy=True),
hg_conf.to(self.device, copy=True),
verbose=False,
bs=eval_bs)
l2d, _ = self.forward(zs, nr)
easy68_err = evaluate(l2d, gt.to(self.device,
copy=True))["with_outline"]
torch.cuda.empty_cache()
# do 68LM hard
d = self.alpha_hg_data["68_lm"]["hard"]
gt, hg_pred, hg_conf = d["gt"], d["hg_pred"], d["hg_conf"]
zs, nr, easy_loss, zs_best, nr_best, best_epochs_easy, sample_losses, _, _ = self.test(
hg_pred.to(self.device, copy=True),
hg_conf.to(self.device, copy=True),
verbose=False,
bs=eval_bs)
l2d, _ = self.forward(zs, nr)
hard68_err = evaluate(l2d, gt.to(self.device,
copy=True))["with_outline"]
torch.cuda.empty_cache()
else:
hard68_err = 100000.0
easy68_err = 100000.0
return {
"e49": easy49_err,
"h49": hard49_err,
"e68": easy68_err,
"h68": hard68_err
}
hg_coords_easy, hg_coords_and_conf_easy, gt_easy, l2d_easy, history_easy = run_pdm(
pdm_path=pdm_path,
hg_results=hg_results["easy"],
location=location,
encoder=args.encoder,
history=needs_history,
bs=args.bs)
hg_coords_hard, hg_coords_and_conf_hard, gt_hard, l2d_hard, history_hard = run_pdm(
pdm_path=pdm_path,
hg_results=hg_results["hard"],
location=location,
encoder=args.encoder,
history=needs_history,
bs=args.bs)
res_easy_before = evaluate(hg_coords_easy, gt_easy)
easy_with_outline_before = res_easy_before["with_outline"]
easy_without_outline_before = res_easy_before["without_outline"]
res_easy_after = evaluate(l2d_easy, gt_easy)
easy_with_outline_after = res_easy_after["with_outline"]
easy_without_outline_after = res_easy_after["without_outline"]
res_hard_before = evaluate(hg_coords_hard, gt_hard)
hard_with_outline_before = res_hard_before["with_outline"]
hard_without_outline_before = res_hard_before["without_outline"]
res_hard_after = evaluate(l2d_hard, gt_hard)
hard_with_outline_after = res_hard_after["with_outline"]
hard_without_outline_after = res_hard_after["without_outline"]
print("[before PDM] easy without \t", easy_without_outline_before)
print("[before PDM] easy with \t\t", easy_with_outline_before)
