def run_e2e_split(e2e, data, location): e2e.eval() images = [] gts = [] hg_preds = [] pdm_preds = [] pdm_3d = [] pdm_applied = [] pdm_encoder_preds = [] for batch in data: image = move2device(batch['image'].float(), location) gt = move2device(batch['landmarks'].float(), location) with torch.no_grad(): pdm_pred, hg_pred, pdm_res_3d, pdm_zs, pdm_nr, apply_pdm, pdm_encoder_pred = e2e(image) images.append(image) gts.append(gt) hg_preds.append(hg_pred) pdm_preds.append(pdm_pred) pdm_3d.append(pdm_res_3d) pdm_applied.append(apply_pdm) if pdm_encoder_pred is not None: pdm_encoder_preds.append(pdm_encoder_pred) images = torch.cat(images) gts = torch.cat(gts) hg_preds = torch.cat(hg_preds) pdm_preds = torch.cat(pdm_preds) pdm_3d = torch.cat(pdm_3d) pdm_applied = torch.cat(pdm_applied) eval_hg = evaluate(hg_preds, gts) eval_pdm = evaluate(pdm_preds, gts) res = { "images": images, "gt": gts, "hg_pred": hg_preds, "pdm_pred": pdm_pred, "pdm_3d": pdm_3d, "eval_hg": eval_hg, "eval_pdm": eval_pdm, "pdm_applied": pdm_applied } if len(pdm_encoder_preds) > 0: pdm_encoder_preds = torch.cat(pdm_encoder_preds) eval_pdm_encoder = evaluate(pdm_encoder_pred, gts) res["pdm_encoder_pred"] = pdm_encoder_preds res["eval_pdm_encoder"] = eval_pdm_encoder return res
def get_errors(source, split): with open(source) as f: data = json.load(f) preds = [] gts = [] errors49 = [] errors68 = [] for i, sample in enumerate(data[split]["results"]): pred = [] gt = [] for landmark in sample["coord_and_variance"]: pred.append([landmark["pred_x"], landmark["pred_y"]]) gt.append([landmark["gt_x"], landmark["gt_y"]]) vx.append(landmark["var_x"]) vy.append(landmark["var_y"]) preds.append(pred) gts.append(gt) if split == "menpo": outline_error, no_outline_error = evaluate_menpo( torch.tensor([pred]), menpo_gt[i:i + 1]) # we need 68 LM to compute the error errors49.append(no_outline_error / 100) errors68.append(outline_error / 100) else: error = evaluate(torch.tensor([pred]), torch.tensor([gt])) errors49.append(error["without_outline"] / 100) errors68.append(error["with_outline"] / 100) print("NO OUTLINE", np.median(errors49)) print("OUTLINE", np.median(errors68)) return errors49, errors68
def run(pdm, hg_results, gpu): location = 'cpu' if gpu is None else "cuda:%d" % gpu data = torch.load(pdm, map_location='cpu') state_dict = data['state_dict'] config = data['config'] make_deterministic(config['random_seed']) net = ModelTrainer.create_net(config) net.model.load_state_dict(state_dict) net.model.eval() net.to(location) net.bs *= 256 hg_out = json.load(open(hg_results, "r")) #avg_dist = torch.tensor(hg_out["train"]["average_lm_distances"], device=location) easy = [x["coord_and_conf"] for x in hg_out["easy"]["results"]] easy_gt = torch.tensor([[[y["gt_x"], y["gt_y"]] for y in x] for x in easy], device=location) hard = [x["coord_and_conf"] for x in hg_out["hard"]["results"]] hard_gt = torch.tensor([[[y["gt_x"], y["gt_y"]] for y in x] for x in hard], device=location) train = [x["coord_and_conf"] for x in hg_out["train"]["results"]] #gauss = norm(0.0, stddev) #easy_hg_pred = torch.tensor([[[y["pred_x"], y["pred_y"], gauss.pdf(y["dist_x"]), gauss.pdf(y["dist_y"])] for y in x] for x in easy], device=location) #hard_hg_pred = torch.tensor([[[y["pred_x"], y["pred_y"], gauss.pdf(y["dist_x"]), gauss.pdf(y["dist_y"])] for y in x] for x in hard], device=location) import math import random #mp = lambda x: (-5494.5 * x + 1.099)**2 #mp = lambda x: 1/(100000*x**2+1) mp = lambda x: min(1, max(0, 1 / x - 130)) mp = lambda x: 1 / x #print(torch.min(avg_dist), torch.max(avg_dist)) #exit() """ varx = torch.tensor([[1/y["var_x"] for y in x] for x in easy], device=location) vary = torch.tensor([[1/y["var_y"] for y in x] for x in easy], device=location) print("easy", torch.min(varx), torch.max(varx)) print("easy", torch.min(vary), torch.max(vary)) varx = torch.tensor([[1/y["var_x"] for y in x] for x in hard], device=location) vary = torch.tensor([[1/y["var_y"] for y in x] for x in hard], device=location) print("hard", torch.min(varx), torch.max(varx)) print("hard", torch.min(vary), torch.max(vary)) varx = torch.tensor([[1/y["var_x"] for y in x] for x in train], device=location) vary = torch.tensor([[1/y["var_y"] for y in x] for x in train], device=location) print("train", torch.min(varx), torch.max(varx)) print("train", torch.min(vary), torch.max(vary)) exit() """ #easy_hg_pred = torch.tensor([[[y["pred_x"], y["pred_y"], mp(avg_dist[i][0]), mp(avg_dist[i][1])] for i,y in enumerate(x)] for x in easy], device=location) #hard_hg_pred = torch.tensor([[[y["pred_x"], y["pred_y"], mp(avg_dist[i][0]), mp(avg_dist[i][1])] for i,y in enumerate(x)] for x in hard], device=location) easy_hg_pred = torch.tensor( [[[y["pred_x"], y["pred_y"], mp(y["var_x"]), mp(y["var_y"])] for i, y in enumerate(x)] for x in easy], device=location) hard_hg_pred = torch.tensor( [[[y["pred_x"], y["pred_y"], mp(y["var_x"]), mp(y["var_y"])] for i, y in enumerate(x)] for x in hard], device=location) #print(torch.min(easy_hg_pred[:,:,2:]), torch.max(easy_hg_pred[:,:,2:])) #print(torch.min(hard_hg_pred[:, :, 2:]), torch.max(hard_hg_pred[:, :, 2:])) #exit() sample_losses_hg_easy = [ np.mean((easy_hg_pred[i, :, :2].cpu().numpy() - easy_gt[i].cpu().numpy())**2) for i in range(easy_gt.shape[0]) ] # TODO test() takes pred and conf now separately zs, nr, *_ = net.test(easy_hg_pred, verbose=True) l2d_easy, _ = net.forward(zs, nr) sample_losses_pdm_easy = [ np.mean((l2d_easy[i].detach().cpu().numpy() - easy_gt[i].detach().cpu().numpy())**2) for i in range(easy_gt.shape[0]) ] easy_best = Counter() best_coords_easy = [] worst_coords_easy = [] for i in range(easy_gt.shape[0]): if sample_losses_pdm_easy[i] <= sample_losses_hg_easy[i]: easy_best["pdm"] += 1 best_coords_easy.append( l2d_easy[i].cpu().detach().numpy().tolist()) worst_coords_easy.append( easy_hg_pred[i, :, :2].cpu().detach().numpy().tolist()) else: easy_best["hg"] += 1 best_coords_easy.append( easy_hg_pred[i, :, :2].cpu().detach().numpy().tolist()) worst_coords_easy.append( l2d_easy[i].cpu().detach().numpy().tolist()) sample_losses_hg_hard = [ np.mean((hard_hg_pred[i, :, :2].cpu().numpy() - hard_gt[i].cpu().numpy())**2) for i in range(hard_gt.shape[0]) ] # TODO test() takes pred and conf now separately zs, nr, *_ = net.test(hard_hg_pred, verbose=True) l2d_hard, _ = net.forward(zs, nr) sample_losses_pdm_hard = [ np.mean((l2d_hard[i].detach().cpu().numpy() - hard_gt[i].detach().cpu().numpy())**2) for i in range(hard_gt.shape[0]) ] hard_best = Counter() best_coords_hard = [] worst_coords_hard = [] for i in range(hard_gt.shape[0]): if sample_losses_pdm_hard[i] <= sample_losses_hg_hard[i]: hard_best["pdm"] += 1 best_coords_hard.append( l2d_hard[i].cpu().detach().numpy().tolist()) worst_coords_hard.append( hard_hg_pred[i, :, :2].cpu().detach().numpy().tolist()) else: hard_best["hg"] += 1 best_coords_hard.append( hard_hg_pred[i, :, :2].cpu().detach().numpy().tolist()) worst_coords_hard.append( l2d_hard[i].cpu().detach().numpy().tolist()) hg_easy_eval = evaluate(easy_hg_pred[:, :, :2], easy_gt) all_pdm_easy_eval = evaluate(l2d_easy, easy_gt) best_pick_easy = evaluate( torch.tensor(best_coords_easy, dtype=torch.float32).cpu(), easy_gt.cpu()) worst_pick_easy = evaluate( torch.tensor(worst_coords_easy, dtype=torch.float32).cpu(), easy_gt.cpu()) print("\n---- EASY without outline----") print("HG \t\t %0.4f" % hg_easy_eval["without_outline"]) print("best pick \t %0.4f" % best_pick_easy["without_outline"]) print("worst pick \t %0.4f" % worst_pick_easy["without_outline"]) print("all PDM \t %0.4f" % all_pdm_easy_eval["without_outline"]) print("\n---- EASY with outline----") print("HG \t\t %0.4f" % hg_easy_eval["with_outline"]) print("best pick \t %0.4f" % best_pick_easy["with_outline"]) print("worst pick \t %0.4f" % worst_pick_easy["with_outline"]) print("all PDM \t %0.4f" % all_pdm_easy_eval["with_outline"]) print("easy best", easy_best) hg_hard_eval = evaluate(hard_hg_pred[:, :, :2], hard_gt) all_pdm_hard_eval = evaluate(l2d_hard, hard_gt) best_pick_hard = evaluate( torch.tensor(best_coords_hard, dtype=torch.float32).cpu(), hard_gt.cpu()) worst_pick_hard = evaluate( torch.tensor(worst_coords_hard, dtype=torch.float32).cpu(), hard_gt.cpu()) print("\n---- HARD without outline----") print("HG \t\t %0.4f" % hg_hard_eval["without_outline"]) print("best pick \t %0.4f" % best_pick_hard["without_outline"]) print("worst pick \t %0.4f" % worst_pick_hard["without_outline"]) print("all PDM \t %0.4f" % all_pdm_hard_eval["without_outline"]) print("\n---- HARD with outline----") print("HG \t\t %0.4f" % hg_hard_eval["with_outline"]) print("best pick \t %0.4f" % best_pick_hard["with_outline"]) print("worst pick \t %0.4f" % worst_pick_hard["with_outline"]) print("all PDM \t %0.4f" % all_pdm_hard_eval["with_outline"]) print("hard_best", hard_best)
def run(self): self.config["model_dir"] = self.model_dir make_deterministic(self.config['random_seed']) pdm = ModelTrainer.create_net(self.config) self.to_gpu(pdm) pdm.verbose = not self.is_gridsearch pdm.listener = self.receive_pdm_output dt = h5py.File(self.data, "r") data_tr = self.to_gpu( torch.tensor(dt["300W"]["train_y"], dtype=torch.float32)) data_te = self.to_gpu( torch.tensor(dt["300W"]["test_y"], dtype=torch.float32)) if self.config["add_multipie"]: tmp = self.to_gpu( torch.tensor(dt["multipie"]["train_y"], dtype=torch.float32)) data_tr = torch.cat((data_tr, tmp)) #print("train", data_tr.shape) #print("test", data_te.shape) #exit() zs_tr, nr_tr, loss_tr = pdm.train(data=data_tr) train_reconstructed, _ = pdm.forward(zs_tr, nr_tr) zs_te, nr_te, loss_te, *_ = pdm.test(data=data_te, confidence=None) test_reconstructed, _ = pdm.forward(zs_te, nr_te) target_file = os.path.join( self.result_dir, "zs_and_nr_%d.json" % self.config["config_id"]) json.dump( { "train": { "zs": zs_tr.detach().cpu().numpy().tolist(), "nr": nr_tr.detach().cpu().numpy().tolist(), "reconstructed": train_reconstructed.detach().cpu().numpy().tolist(), "coords": data_tr.detach().cpu().numpy().tolist() }, "test": { "zs": zs_te.detach().cpu().numpy().tolist(), "nr": nr_te.detach().cpu().numpy().tolist(), "reconstructed": test_reconstructed.detach().cpu().numpy().tolist(), "coords": data_te.detach().cpu().numpy().tolist() } }, open(target_file, "w")) pdm.save_pdm( pdm.train_epochs, os.path.join(self.model_dir, "final_pdm_%d.torch" % self.config["config_id"])) # TODO train ENCODERS DIRECTLY HERE if self.is_gridsearch: last_train_loss = self.loss_log["train"][-1] lowest_train_loss = min(self.loss_log["train"]) best_train_epoch = min([ i for i in range(len(self.loss_log["train"])) if self.loss_log["train"][i] == lowest_train_loss ]) train_error = evaluate(train_reconstructed, data_tr) test_error = evaluate(test_reconstructed, data_te) #print(train_error, test_error) best_epochs = { "best_%s_epoch" % k: v for k, v in self.best_epoch.items() } best_errors = { "best_%s" % k: v for k, v in self.lowest_error.items() } return { **self.config, "last_train_loss": last_train_loss, "lowest_train_loss": lowest_train_loss, "best_train_epoch": best_train_epoch, "metrics_log": self.metrics_log, **best_epochs, **best_errors, "train_error_49": train_error["without_outline"], "train_error_68": train_error["with_outline"], "test_error_49": test_error["without_outline"], "test_error_68": test_error["with_outline"] } else: # evaluate PDM metrics = pdm.eval_on_alpha_hg() print(metrics["easy_metrics_last"]) print(metrics["hard_metrics_last"])
gt_lm = batch['landmarks'] img = batch['image'].to(location) starttime = time.time() predicted_landmarks, heatmaps, var, unnormal_hms = net(img) endtime = time.time() duration_batch = endtime - starttime durations.append(duration_batch) #print(duration_batch) gts.append(gt_lm.detach()) preds.append(predicted_landmarks.detach()) print("mean duration per batch", sum(durations) / len(durations)) print() print("Menpo with BB size error") res_menpo = evaluate_menpo( torch.cat(preds).cpu().float(), torch.cat(gts).cpu().float()) print("Without outline", res_menpo[1]) print("With outline", res_menpo[0]) print() res_menpo_iod = evaluate( torch.cat(preds).cpu().float(), torch.cat(gts).cpu().float()) print("menpo with IOD error") print("49", res_menpo_iod["without_outline"]) print("68", res_menpo_iod["with_outline"])
tmp.append((xs[i][k].item(), ys[i][k].item())) coordpreds.append(tmp) ibug_pred = coordpreds[0:135] lfpw_pred = coordpreds[135:359] helen_pred = coordpreds[359:689] helen_lfpw_pred = helen_pred + lfpw_pred gt_data = sio.loadmat(gt_mat_src)["W300_gt"].transpose(2, 1, 0).tolist() ibug_gt = gt_data[0:135] lfpw_gt = gt_data[135:359] helen_gt = gt_data[359:689] helen_lfpw_gt = helen_gt + lfpw_gt res_ibug = evaluate( torch.tensor(ibug_pred).float(), torch.tensor(ibug_gt).float()) print("ibug", res_ibug) res_lfpw = evaluate( torch.tensor(lfpw_pred).float(), torch.tensor(lfpw_gt).float()) print("lfpw", res_lfpw) res_helen = evaluate( torch.tensor(helen_pred).float(), torch.tensor(helen_gt).float()) print("helen", res_helen) res_helen_lfpw = evaluate( torch.tensor(helen_lfpw_pred).float(),
def evaluate_predictions_split(split): gt = torch.tensor(split["gt"]) pred = torch.tensor(split["pdm_pred"]) res = evaluate(pred, gt) return res
def eval_on_alpha_hg(self): self.init_hg_results() eval_bs = 2048 # do 49LM easy d = self.alpha_hg_data["49_lm"]["easy"] gt, hg_pred, hg_conf = d["gt"], d["hg_pred"], d["hg_conf"] zs, nr, easy_loss, zs_best, nr_best, best_epochs_easy, sample_losses, _, _ = self.test( hg_pred.to(self.device, copy=True), hg_conf.to(self.device, copy=True), verbose=False, bs=eval_bs) l2d, _ = self.forward(zs, nr) easy49_err = evaluate(l2d, gt.to(self.device, copy=True))["without_outline"] torch.cuda.empty_cache() # do 49LM hard d = self.alpha_hg_data["49_lm"]["hard"] gt, hg_pred, hg_conf = d["gt"], d["hg_pred"], d["hg_conf"] zs, nr, easy_loss, zs_best, nr_best, best_epochs_easy, sample_losses, _, _ = self.test( hg_pred.to(self.device, copy=True), hg_conf.to(self.device, copy=True), verbose=False, bs=eval_bs) l2d, _ = self.forward(zs, nr) hard49_err = evaluate(l2d, gt.to(self.device, copy=True))["without_outline"] torch.cuda.empty_cache() if self.is_68_pdm: # do 68LM easy d = self.alpha_hg_data["68_lm"]["easy"] gt, hg_pred, hg_conf = d["gt"], d["hg_pred"], d["hg_conf"] zs, nr, easy_loss, zs_best, nr_best, best_epochs_easy, sample_losses, _, _ = self.test( hg_pred.to(self.device, copy=True), hg_conf.to(self.device, copy=True), verbose=False, bs=eval_bs) l2d, _ = self.forward(zs, nr) easy68_err = evaluate(l2d, gt.to(self.device, copy=True))["with_outline"] torch.cuda.empty_cache() # do 68LM hard d = self.alpha_hg_data["68_lm"]["hard"] gt, hg_pred, hg_conf = d["gt"], d["hg_pred"], d["hg_conf"] zs, nr, easy_loss, zs_best, nr_best, best_epochs_easy, sample_losses, _, _ = self.test( hg_pred.to(self.device, copy=True), hg_conf.to(self.device, copy=True), verbose=False, bs=eval_bs) l2d, _ = self.forward(zs, nr) hard68_err = evaluate(l2d, gt.to(self.device, copy=True))["with_outline"] torch.cuda.empty_cache() else: hard68_err = 100000.0 easy68_err = 100000.0 return { "e49": easy49_err, "h49": hard49_err, "e68": easy68_err, "h68": hard68_err }
hg_coords_easy, hg_coords_and_conf_easy, gt_easy, l2d_easy, history_easy = run_pdm( pdm_path=pdm_path, hg_results=hg_results["easy"], location=location, encoder=args.encoder, history=needs_history, bs=args.bs) hg_coords_hard, hg_coords_and_conf_hard, gt_hard, l2d_hard, history_hard = run_pdm( pdm_path=pdm_path, hg_results=hg_results["hard"], location=location, encoder=args.encoder, history=needs_history, bs=args.bs) res_easy_before = evaluate(hg_coords_easy, gt_easy) easy_with_outline_before = res_easy_before["with_outline"] easy_without_outline_before = res_easy_before["without_outline"] res_easy_after = evaluate(l2d_easy, gt_easy) easy_with_outline_after = res_easy_after["with_outline"] easy_without_outline_after = res_easy_after["without_outline"] res_hard_before = evaluate(hg_coords_hard, gt_hard) hard_with_outline_before = res_hard_before["with_outline"] hard_without_outline_before = res_hard_before["without_outline"] res_hard_after = evaluate(l2d_hard, gt_hard) hard_with_outline_after = res_hard_after["with_outline"] hard_without_outline_after = res_hard_after["without_outline"] print("[before PDM] easy without \t", easy_without_outline_before) print("[before PDM] easy with \t\t", easy_with_outline_before)